import sys
import gym

from elegantrl.train.run import train_and_evaluate, train_and_evaluate_mp
from elegantrl.train.config import Arguments
from elegantrl.agents.AgentDDPG import AgentDDPG, AgentDDPGHterm
from elegantrl.agents.AgentTD3 import AgentTD3
from elegantrl.agents.AgentSAC import AgentSAC, AgentReSAC, AgentReSACHterm, AgentReSACHtermK


def demo_ddpg_td3_sac(gpu_id, drl_id, env_id):  # 2022.02.02
    env_name = ['Pendulum-v0',
                'Pendulum-v1',
                'LunarLanderContinuous-v2',
                'BipedalWalker-v3',
                'Hopper-v2',
                'HalfCheetah-v2'
                'Humanoid-v3', ][env_id]
    agent = [AgentDDPG, AgentTD3, AgentSAC, AgentReSAC][drl_id]

    if env_name in {'Pendulum-v0', 'Pendulum-v1'}:
        from elegantrl.envs.CustomGymEnv import PendulumEnv
        env = PendulumEnv(env_name, target_return=-500)
        """
        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        2  1.60e+03-1147.49 |-1147.49  179.2    200     0 |   -2.61   0.90   0.55   1.00
        2  5.84e+04 -121.61 | -121.61   59.0    200     0 |   -0.81   0.33 -40.64   0.79
        | UsedTime:     132 |

        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        1  1.60e+03-1267.96 |-1267.96  329.7    200     0 |   -2.67   0.88   0.56   1.00
        1  8.48e+04 -171.79 | -182.24   63.3    200     0 |   -0.30   0.32 -30.75   0.64
        1  1.19e+05 -171.79 | -178.25  116.8    200     0 |   -0.31   0.16 -22.52   0.43
        1  1.34e+05 -164.56 | -164.56   99.1    200     0 |   -0.31   0.15 -18.09   0.35
        1  1.47e+05 -135.20 | -135.20   92.1    200     0 |   -0.31   0.14 -15.65   0.29
        | UsedTime:     783 | 
        """
        args = Arguments(agent, env)
        args.reward_scale = 2 ** -1  # RewardRange: -1800 < -200 < -50 < 0
        args.gamma = 0.97
        args.target_step = args.max_step * 2
        args.eval_times = 2 ** 3
    elif env_name == 'LunarLanderContinuous-v2':
        """
        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        2  4.25e+03 -143.93 | -143.93   29.6     69    12 |   -2.47   1.06   0.13   0.15
        2  1.05e+05  170.35 |  170.35   57.9    645   177 |    0.06   1.59  15.93   0.20
        2  1.59e+05  170.35 |   80.46  125.0    775   285 |    0.07   1.14  29.92   0.29
        2  1.95e+05  221.39 |  221.39   19.7    449   127 |    0.12   1.09  32.16   0.40
        | UsedTime:     421 |

        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        1  4.26e+03 -139.77 | -139.77   36.7     67    12 |   -2.16  11.20   0.12   0.15
        1  1.11e+05 -105.09 | -105.09   84.3    821   244 |   -0.14  27.60   1.04   0.21
        1  2.03e+05  -15.21 |  -15.21   22.7   1000     0 |   -0.01  17.96  36.95   0.45
        1  3.87e+05   59.39 |   54.09  160.7    756   223 |    0.00  16.57  88.99   0.73
        1  4.03e+05   59.39 |   56.16  103.5    908   120 |    0.06  16.47  84.27   0.71
        1  5.10e+05  186.59 |  186.59  103.6    547   257 |   -0.02  12.72  67.97   0.57
        1  5.89e+05  226.93 |  226.93   20.0    486   154 |    0.13   9.27  68.29   0.51
        | UsedTime:    3407 |

        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        1  4.15e+03 -169.01 | -169.01   87.9    110    59 |   -2.18  11.86   0.10   0.15
        1  1.09e+05  -84.47 |  -84.47   80.1    465   293 |   -0.30  30.64  -6.29   0.20
        1  4.25e+05   -8.33 |   -8.33   48.4    994    26 |    0.07  13.51  76.99   0.62
        1  4.39e+05   87.29 |   87.29   86.9    892   141 |    0.04  12.76  70.37   0.61
        1  5.57e+05  159.17 |  159.17   65.7    721   159 |    0.10  10.31  59.90   0.51
        1  5.87e+05  190.09 |  190.09   71.7    577   175 |    0.09   9.45  61.74   0.48
        1  6.20e+05  206.74 |  206.74   29.1    497   108 |    0.09   9.21  62.06   0.47
        | UsedTime:    4433 |
        """
        env_func = gym.make
        env_args = {'env_num': 1,
                    'env_name': 'LunarLanderContinuous-v2',
                    'max_step': 1000,
                    'state_dim': 8,
                    'action_dim': 2,
                    'if_discrete': False,
                    'target_return': 200,

                    'id': 'LunarLanderContinuous-v2'}
        args = Arguments(agent, env_func=env_func, env_args=env_args)

        args.target_step = args.max_step
        args.gamma = 0.99
        args.eval_times = 2 ** 5
    elif env_name == 'BipedalWalker-v3':
        env_func = gym.make
        env_args = {'env_num': 1,
                    'env_name': 'BipedalWalker-v3',
                    'max_step': 1600,
                    'state_dim': 24,
                    'action_dim': 4,
                    'if_discrete': False,
                    'target_return': 300,

                    'id': 'BipedalWalker-v3', }
        args = Arguments(agent, env_func=env_func, env_args=env_args)
        args.target_step = args.max_step
        args.gamma = 0.98
        args.eval_times = 2 ** 4

        """
        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        3  7.51e+03 -111.59 | -111.59    0.2     97     7 |   -0.18   4.23  -0.03   0.02
        3  1.48e+05 -110.19 | -110.19    1.6     84    30 |   -0.59   2.46   3.18   0.03
        3  5.02e+05  -31.84 | -102.27   54.0   1359   335 |   -0.06   0.85   2.84   0.04
        3  1.00e+06   -7.94 |   -7.94   73.2    411   276 |   -0.17   0.72   1.96   0.03
        3  1.04e+06  131.50 |  131.50  168.3    990   627 |    0.06   0.46   1.69   0.04
        3  1.11e+06  214.12 |  214.12  146.6   1029   405 |    0.09   0.50   1.63   0.04
        3  1.20e+06  308.34 |  308.34    0.7   1106    20 |    0.29   0.72   4.56   0.05
        | UsedTime:    8611 |

        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        3  6.75e+03  -92.44 |  -92.44    0.2    120     3 |   -0.18   1.94  -0.00   0.02
        3  3.95e+05  -37.16 |  -37.16    9.2   1600     0 |   -0.06   1.90   4.20   0.07
        3  6.79e+05  -23.32 |  -42.54   90.0   1197   599 |   -0.02   0.91   1.57   0.04
        3  6.93e+05   46.92 |   46.92   96.9    808   395 |   -0.04   0.57   1.34   0.04
        3  8.38e+05  118.86 |  118.86  154.5    999   538 |    0.14   1.44   0.75   0.05
        3  1.00e+06  225.56 |  225.56  124.1   1207   382 |    0.13   0.72   4.75   0.06
        3  1.02e+06  283.37 |  283.37   86.3   1259   245 |    0.14   0.80   3.96   0.06
        3  1.19e+06  313.36 |  313.36    0.9   1097    20 |    0.21   0.78   6.80   0.06
        | UsedTime:    9354 | SavedDir: ./BipedalWalker-v3_ModSAC_3

        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        3  6.55e+03 -109.86 | -109.86    4.5    156    30 |   -0.06   0.71  -0.01   0.02
        3  1.24e+05  -88.28 |  -88.28   26.2    475   650 |   -0.15   0.15   0.04   0.02
        3  3.01e+05  -47.89 |  -56.76   21.7   1341   540 |   -0.03   0.19  -2.76   0.05
        3  3.82e+05   80.89 |   53.79  140.1    983   596 |   -0.01   0.18   0.46   0.05
        3  4.35e+05  137.70 |   28.54  104.7    936   581 |   -0.01   0.21   0.63   0.06
        3  4.80e+05  158.71 |   25.54  114.7    524   338 |    0.18   0.17   6.17   0.06
        3  5.31e+05  205.81 |  203.27  143.9   1048   388 |    0.14   0.15   4.00   0.06
        3  6.93e+05  254.40 |  252.74  121.1    992   280 |    0.21   0.12   7.34   0.06
        3  7.11e+05  304.79 |  304.79   73.4   1015   151 |    0.21   0.12   5.69   0.06
        | UsedTime:    3215 | 

        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        1  7.08e+03 -106.48 | -106.48    6.0    170    17 |   -0.14   0.70   0.03   0.02
        1  2.38e+05  -89.62 |  -89.62   29.8    775   728 |   -0.30   0.31 -13.44   0.04
        1  4.12e+05  -33.40 |  -34.50   27.6   1342   516 |   -0.01   0.20   1.34   0.06
        1  5.05e+05    2.54 |  -47.29   20.9   1342   516 |    0.02   0.17   0.24   0.05
        1  5.43e+05   52.93 |   52.93  107.6   1084   540 |   -0.21   0.15   0.32   0.05
        1  5.80e+05  138.30 |  136.60   77.6   1460   176 |    0.10   0.16   2.14   0.05
        1  6.16e+05  188.98 |  171.72   99.2   1386   305 |    0.12   0.16  -0.40   0.05
        1  7.06e+05  250.72 |  231.97  142.9   1247   448 |    0.12   0.13   2.81   0.05
        1  8.06e+05  287.28 |  -68.06    5.9    211    19 |   -0.08   0.12   7.83   0.06
        1  8.56e+05  291.10 |  286.19   56.0   1181    63 |    0.17   0.13   6.37   0.06
        1  8.83e+05  314.54 |  314.54    1.0   1252    19 |    0.11   0.12   7.23   0.06
        | UsedTime:    5008 | 
        """

        """
        | Arguments Remove cwd: ./BipedalWalker-v3_ReliableSAC_2
################################################################################
ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
2  7.27e+03 -117.98 |
2  7.27e+03 -117.98 | -117.98    2.5     43     6 |   -0.09   0.62   0.03   0.02
2  1.03e+05 -102.54 |
2  1.03e+05 -102.54 | -102.54    2.3     71     6 |   -0.97   0.11  -1.68   0.02
2  1.51e+05  -46.60 |
2  1.51e+05  -46.60 |  -46.60    8.4   1600     0 |   -0.05   0.62  -8.64   0.07
2  1.89e+05  -41.07 |
2  1.89e+05  -41.07 |  -41.07    0.0   1600     0 |   -0.08   0.36   0.74   0.05
2  2.20e+05  -30.91 |
2  2.20e+05  -30.91 |  -30.91    1.7   1600     0 |   -0.11   0.23   0.60   0.04
2  2.48e+05  -30.91 |  -59.86   16.5   1600     0 |   -0.04   0.17   0.45   0.03
2  2.73e+05  -30.91 |  -66.04   29.2    713   724 |   -0.03   0.12  -0.53   0.03
2  2.97e+05  -30.91 |  -90.05   35.6    783   485 |   -0.10   0.16  -1.07   0.03
2  3.22e+05  -30.91 |  -38.24   41.4    507   251 |   -0.18   0.15  -3.86   0.04
2  3.48e+05  235.88 |
2  3.48e+05  235.88 |  235.88  124.6   1210   376 |   -0.06   0.17  -2.53   0.04
2  3.70e+05  235.88 |  -79.56    3.8    138    12 |    0.08   0.19   0.85   0.05
2  3.88e+05  235.88 |  171.11  174.6    933   517 |   -0.14   0.21   2.51   0.05
2  4.08e+05  322.22 |
2  4.08e+05  322.22 |  322.22    0.6   1291    12 |   -0.39   0.20   0.15   0.06
| UsedTime:    1686 | SavedDir: ./BipedalWalker-v3_ReliableSAC_2
| Learner: Save in ./BipedalWalker-v3_ReliableSAC_2
| LearnerPipe.run: ReplayBuffer saving in ./BipedalWalker-v3_ReliableSAC_2

        """
    elif env_name == 'Hopper-v2':
        env_func = gym.make
        env_args = {
            'env_num': 1,
            'env_name': 'Hopper-v2',
            'max_step': 1000,
            'state_dim': 11,
            'action_dim': 3,
            'if_discrete': False,
            'target_return': 3800.,
        }
        args = Arguments(agent, env_func=env_func, env_args=env_args)
        args.eval_times = 2 ** 2
        args.reward_scale = 2 ** -4

        args.target_step = args.max_step * 2
        args.worker_num = 2

        args.net_dim = 2 ** 8
        args.num_layer = 3
        args.batch_size = int(args.net_dim * 2)
        args.repeat_times = 2 ** 4
        args.gamma = 0.993  # todo

        args.if_allow_break = False
        args.break_step = int(8e6)

        """
        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        5  1.61e+04  131.99 |  131.99    3.6     81     2 |    0.03   0.09   0.03  -0.54
        5  2.20e+05  391.44 |  391.44    0.3    158     0 |    0.08   0.01  -0.06  -0.75
        5  4.25e+05  860.96 |  860.96   11.9    280     5 |    0.09   0.11   0.12  -0.84
        5  6.27e+05 3001.43 | 3001.43    7.9   1000     0 |    0.10   0.78  -0.01  -0.85
        5  1.64e+06 3203.09 | 3103.14    0.0   1000     0 |    0.10   1.82  -0.06  -0.76
        5  2.86e+06 3256.43 | 3152.72    0.0   1000     0 |    0.10   0.75   0.01  -0.67
        5  3.88e+06 3256.43 | 1549.69    0.0    512     0 |    0.10   0.86   0.00  -0.71
        | UsedTime:    2565 |

        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        2  1.60e+04  328.68 |  328.68    6.2    262     6 |    0.02   0.01  -0.02  -0.54
        2  2.16e+05 2460.57 | 2460.57   14.3   1000     0 |    0.09   0.86   0.20  -0.74
        2  6.22e+05 2789.97 | 2788.28   30.9   1000     0 |    0.10   0.40  -0.11  -1.04
        2  1.23e+06 3263.16 | 3216.96    0.0   1000     0 |    0.10   1.06   0.12  -1.05
        2  2.46e+06 3378.50 | 3364.02    0.0   1000     0 |    0.11   0.87   0.02  -0.92
        2  3.90e+06 3397.88 | 3302.80    0.0   1000     0 |    0.11   0.46   0.01  -0.93
        | UsedTime:    2557 |

        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        4  2.41e+04  222.39 |  222.39    1.5    120     1 |    0.94   8.45   0.05  -0.55
        4  5.34e+05  344.58 |  344.58    0.4    142     0 |    2.41   1.91   0.02  -0.94
        4  8.74e+05  540.69 |  540.69   20.1    180     4 |    2.96   5.82   0.00  -1.10
        4  1.39e+06  989.51 |  989.51    2.2    308     2 |    3.20  16.75   0.07  -1.08
        4  1.73e+06 3161.60 | 3149.35    0.0   1000     0 |    3.26  43.84  -0.02  -1.08
        4  2.06e+06 3367.27 | 3105.77    0.0   1000     0 |    3.32  44.14   0.00  -1.13
        4  3.92e+06 3604.42 | 3565.39    0.0   1000     0 |    3.44  30.54   0.04  -1.04
        4  5.76e+06 3717.06 | 3607.94    0.0   1000     0 |    3.40  51.92   0.07  -0.95
        4  6.26e+06 3840.95 | 3409.25    0.0   1000     0 |    3.32  66.48  -0.02  -0.94
        | UsedTime:    6251 |
        
        
| Arguments Remove cwd: ./Hopper-v2_PPO_4
################################################################################
ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
4  4.00e+03   80.27 |
4  4.00e+03   80.27 |   80.27    0.7     50     0 |    0.07   0.92   0.07   0.00
4  1.46e+05  408.17 |
4  1.46e+05  408.17 |  408.17    1.5    145     1 |    0.17   0.03   0.13   0.00
4  2.90e+05 1856.89 |
4  2.90e+05 1856.89 | 1856.89  589.7    692   219 |    0.19   1.01  -0.05   0.00
4  4.34e+05 1856.89 | 1768.29    0.0    544     0 |    0.20   0.19   0.08   0.00
4  5.81e+05 2750.76 |
4  5.81e+05 2750.76 | 2750.76   15.5   1000     0 |    0.18   2.98   0.10   0.00
4  7.24e+05 3036.12 |
4  7.24e+05 3036.12 | 3036.12    2.2   1000     0 |    0.19   2.75   0.05   0.00
4  8.70e+05 3036.12 | 2790.94    0.0   1000     0 |    0.19   2.01   0.05   0.00
4  1.01e+06 3220.42 |
4  1.01e+06 3220.42 | 3220.42    2.7   1000     0 |    0.20   1.70   0.05   0.00
4  1.15e+06 3220.42 | 3059.18  312.9    939   105 |    0.20   1.30   0.05   0.00
4  1.30e+06 3220.42 | 2892.04    0.0   1000     0 |    0.19   2.23  -0.06   0.00
4  1.44e+06 3220.42 | 3153.15    0.0   1000     0 |    0.20   1.04   0.05   0.00
4  1.59e+06 3220.42 | 3083.19    0.0   1000     0 |    0.20   1.40   0.05   0.00
4  1.73e+06 3220.42 | 2999.16    0.0   1000     0 |    0.19   2.53   0.03   0.00
4  1.88e+06 3220.42 | 3219.83   31.4   1000     0 |    0.20   1.11   0.04   0.00
4  2.01e+06 3220.42 | 1465.88    0.0    499     0 |    0.19   2.71   0.22   0.00
4  2.16e+06 3220.42 | 3157.03    0.0   1000     0 |    0.21   0.62   0.08   0.00
4  2.30e+06 3220.42 | 1256.07    0.0    379     0 |    0.20   2.89   0.07   0.00
4  2.45e+06 3265.09 |
4  2.45e+06 3265.09 | 3265.09   14.0   1000     0 |    0.19   3.52   0.01   0.00
4  2.59e+06 3265.09 | 1562.53    0.0    498     0 |    0.18   2.96   0.08   0.00
4  2.73e+06 3265.09 | 3238.68    0.0   1000     0 |    0.20   1.73   0.07   0.00
4  2.87e+06 3265.09 | 3240.99    0.0   1000     0 |    0.20   3.32  -0.16   0.00
4  3.02e+06 3265.09 | 3141.53    0.0   1000     0 |    0.19   3.55  -0.04   0.00
4  3.16e+06 3265.09 | 3252.13    0.0   1000     0 |    0.21   1.44  -0.03   0.00
4  3.30e+06 3265.09 | 3164.95    0.0   1000     0 |    0.20   2.48   0.10   0.00
| UsedTime: 3049 | SavedDir: ./Hopper-v2_PPO_4


| Arguments Remove cwd: ./Hopper-v2_ReliableSAC_3
################################################################################
ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
3  8.09e+03   32.31 |
3  8.09e+03   32.31 |   32.31    0.6     29     0 |    0.05   0.12   0.44   0.05
3  4.86e+04  282.45 |
3  4.86e+04  282.45 |  282.45    2.6    112     1 |    0.14   0.03   6.95   0.07
3  6.93e+04  282.45 |   82.75    0.0     56     0 |    0.16   0.54  31.61   0.28
3  9.00e+04  282.45 |  247.28    0.0    124     0 |    0.12   0.53  54.26   0.26
3  1.06e+05  306.45 |
3  1.06e+05  306.45 |  306.45    1.4    120     0 |    0.15   0.50  55.85   0.38
3  1.18e+05  406.01 |
3  1.18e+05  406.01 |  406.01    2.9    177     1 |    0.14   0.59  61.50   0.51
3  1.31e+05  450.03 |
3  1.31e+05  450.03 |  450.03    5.5    170     2 |    0.12   0.72  66.32   0.43
3  1.47e+05  450.03 |  155.34    0.0     88     0 |    0.13   0.90  87.74   0.47
3  1.64e+05  450.03 |  154.29    0.0     85     0 |    0.15   0.84  83.98   0.59
3  1.76e+05  450.03 |  276.25    0.0    132     0 |    0.13   0.91  91.40   0.46
3  1.89e+05 1351.93 |
3  1.89e+05 1351.93 | 1351.93  360.9    686   202 |    0.14   0.81  87.22   0.47
3  2.01e+05 1351.93 |  278.82    0.0    117     0 |    0.07   1.26 105.55   0.74
3  2.14e+05 1351.93 |  891.33    0.0    659     0 |    0.14   1.23 124.07   0.52
3  2.27e+05 1351.93 |  645.46    0.0    252     0 |    0.10   1.24 103.56   0.56
3  2.39e+05 1351.93 |  231.23    0.0    109     0 |    0.11   2.02 151.70   0.45
3  2.52e+05 1351.93 |  603.45    0.0    239     0 |    0.15   1.75 134.23   0.75
3  2.64e+05 1351.93 | 1030.42    0.0   1000     0 |    0.12   1.13 107.67   0.67
3  2.73e+05 1351.93 | 1047.41    0.0   1000     0 |    0.07   0.74  93.73   0.50
3  2.82e+05 1351.93 | 1111.93    0.0   1000     0 |    0.08   0.62  81.83   0.38
3  2.91e+05 1351.93 | 1039.49    0.0   1000     0 |    0.09   0.52  66.07   0.34
3  3.00e+05 1490.20 |
3  3.00e+05 1490.20 | 1490.20    0.7   1000     0 |    0.08   0.34  55.97   0.23
3  3.09e+05 1526.75 |
3  3.09e+05 1526.75 | 1526.75   36.5   1000     0 |    0.10   0.21  41.72   0.17
3  3.18e+05 1526.75 |  240.00    0.0    107     0 |    0.13   1.18  83.82   0.59
3  3.27e+05 1526.75 |  670.09    0.0    268     0 |    0.12   0.81 107.00   0.44
3  3.35e+05 1526.75 |  712.00    0.0    442     0 |    0.16   0.71  81.89   0.33
3  3.43e+05 1526.75 |  544.57    0.0    276     0 |    0.13   0.43  60.21   0.18
3  3.52e+05 1526.75 |  379.76    0.0    136     0 |    0.16   0.59  66.49   0.25
3  3.61e+05 1526.75 | 1025.86    0.0   1000     0 |    0.17   0.77  98.66   0.46
3  3.69e+05 1526.75 | 1039.17    0.0   1000     0 |    0.06   0.66  88.29   0.39
3  3.77e+05 1782.66 |
3  3.77e+05 1782.66 | 1782.66    2.0   1000     0 |    0.10   0.40  65.66   0.25
3  3.86e+05 1782.66 | 1031.96    0.0   1000     0 |    0.13   0.23  45.79   0.16
3  3.94e+05 2410.15 |
3  3.94e+05 2410.15 | 2410.15    4.9   1000     0 |    0.12   0.14  32.34   0.09
3  4.04e+05 2556.36 |
3  4.04e+05 2556.36 | 2556.36    3.4   1000     0 |    0.15   0.09  24.99   0.08
3  4.13e+05 2575.00 |
3  4.13e+05 2575.00 | 2575.00    6.8   1000     0 |    0.17   0.07  22.31   0.07
3  4.21e+05 2575.00 | 1018.50    0.0   1000     0 |    0.16   0.09  29.45   0.11
3  4.27e+05 2575.00 | 1001.91    0.0   1000     0 |    0.12   0.08  27.80   0.11
3  4.31e+05 2575.00 |  863.46    0.0   1000     0 |    0.07   0.08  27.16   0.10
3  4.35e+05 2575.00 | 1058.56    0.0   1000     0 |    0.06   0.10  32.01   0.12
3  4.39e+05 2575.00 | 1038.11    0.0   1000     0 |    0.06   0.09  28.62   0.10
3  4.44e+05 2575.00 | 2124.24    0.0   1000     0 |    0.09   0.07  26.10   0.07
3  4.48e+05 2761.82 |
3  4.48e+05 2761.82 | 2761.82    0.8   1000     0 |    0.07   0.06  24.11   0.07
3  4.52e+05 2761.82 | 2732.77    0.0   1000     0 |    0.14   0.05  23.25   0.06
3  4.56e+05 2822.69 |
3  4.56e+05 2822.69 | 2822.69    0.5   1000     0 |    0.17   0.05  22.39   0.06
3  4.60e+05 2994.40 |
3  4.60e+05 2994.40 | 2994.40    1.9   1000     0 |    0.17   0.04  22.00   0.06
3  4.64e+05 2994.40 | 2824.77    0.0   1000     0 |    0.18   0.05  21.46   0.06
3  4.68e+05 2994.40 | 2826.93    0.0   1000     0 |    0.19   0.04  21.57   0.07
3  4.73e+05 2994.40 | 2963.28    0.0   1000     0 |    0.19   0.04  21.16   0.06
3  4.77e+05 2994.40 | 2928.26    0.0   1000     0 |    0.18   0.04  21.61   0.06
3  4.82e+05 3005.36 |
3  4.82e+05 3005.36 | 3005.36    1.0   1000     0 |    0.19   0.04  21.32   0.06
3  4.86e+05 3141.08 |
3  4.86e+05 3141.08 | 3141.08    1.9   1000     0 |    0.18   0.04  21.49   0.05
3  4.90e+05 3141.08 | 3004.14    0.0   1000     0 |    0.19   0.04  21.65   0.06
3  4.94e+05 3141.08 | 3052.78    0.0   1000     0 |    0.19   0.04  21.54   0.06
3  4.98e+05 3141.08 | 3045.90    0.0   1000     0 |    0.19   0.04  22.11   0.06
3  5.02e+05 3228.76 |
3  5.02e+05 3228.76 | 3228.76    2.8   1000     0 |    0.19   0.04  21.98   0.06
3  5.06e+05 3228.76 | 3173.43    0.0   1000     0 |    0.19   0.04  21.85   0.06
3  5.10e+05 3228.76 | 3154.30    0.0   1000     0 |    0.19   0.04  22.53   0.06
3  5.14e+05 3228.76 | 3207.25    0.0   1000     0 |    0.19   0.04  22.33   0.06
3  5.18e+05 3228.76 | 3131.76    0.0   1000     0 |    0.20   0.04  22.29   0.06
3  5.22e+05 3278.78 |
3  5.22e+05 3278.78 | 3278.78    3.8   1000     0 |    0.20   0.03  22.38   0.06
3  5.27e+05 3278.78 | 3236.48    0.0   1000     0 |    0.19   0.04  22.67   0.05
3  5.31e+05 3278.78 | 3152.81    0.0   1000     0 |    0.20   0.03  22.13   0.05
3  5.35e+05 3278.78 | 3140.72    0.0   1000     0 |    0.20   0.04  21.65   0.05
3  5.39e+05 3278.78 | 3270.06    0.0   1000     0 |    0.20   0.03  22.16   0.05
3  5.43e+05 3278.78 | 3189.89    0.0   1000     0 |    0.20   0.03  22.27   0.05
3  5.48e+05 3278.78 | 3178.73    0.0   1000     0 |    0.20   0.03  22.80   0.05
3  5.52e+05 3278.78 | 3269.01    0.0   1000     0 |    0.20   0.03  22.18   0.05
3  5.56e+05 3318.49 |
3  5.56e+05 3318.49 | 3318.49    1.9   1000     0 |    0.20   0.03  22.41   0.05
3  5.61e+05 3346.12 |
3  5.61e+05 3346.12 | 3346.12    2.3   1000     0 |    0.20   0.03  21.79   0.05
3  5.65e+05 3346.12 | 3285.40    0.0   1000     0 |    0.20   0.03  22.62   0.05
3  5.69e+05 3346.12 | 3253.44    0.0   1000     0 |    0.21   0.03  22.43   0.05
3  5.73e+05 3346.12 | 3235.30    0.0   1000     0 |    0.20   0.03  22.33   0.05
3  5.78e+05 3346.12 | 3119.58    0.0   1000     0 |    0.21   0.04  22.57   0.05
3  5.83e+05 3346.12 | 3171.89    0.0   1000     0 |    0.20   0.04  22.48   0.05
3  5.87e+05 3346.12 | 3185.86    0.0   1000     0 |    0.19   0.03  23.00   0.05
3  5.91e+05 3346.12 | 3240.08    0.0   1000     0 |    0.20   0.03  23.02   0.05
3  5.95e+05 3346.12 | 3204.16    0.0   1000     0 |    0.20   0.03  22.30   0.05
3  5.99e+05 3346.12 | 3270.83    0.0   1000     0 |    0.20   0.03  22.61   0.05
3  6.04e+05 3346.12 | 3241.98    0.0   1000     0 |    0.20   0.03  22.94   0.05
3  6.08e+05 3346.12 | 3179.65    0.0   1000     0 |    0.20   0.03  22.36   0.05
3  6.13e+05 3346.12 | 3216.45    0.0   1000     0 |    0.21   0.03  22.43   0.05
3  6.17e+05 3346.12 | 3282.53    0.0   1000     0 |    0.20   0.03  22.51   0.05
3  6.21e+05 3346.12 | 3301.64    0.0   1000     0 |    0.20   0.03  22.65   0.05
3  6.26e+05 3346.12 | 3233.07    0.0   1000     0 |    0.21   0.03  22.66   0.05
3  6.31e+05 3346.12 | 3285.32    0.0   1000     0 |    0.21   0.03  22.35   0.05
3  6.35e+05 3346.12 | 3244.01    0.0   1000     0 |    0.20   0.03  22.82   0.05
3  6.39e+05 3346.12 | 3210.21    0.0   1000     0 |    0.21   0.03  22.64   0.05
3  6.45e+05 3346.12 | 3209.60    0.0   1000     0 |    0.21   0.03  22.88   0.05
3  6.50e+05 3346.12 | 3259.00    0.0   1000     0 |    0.21   0.03  22.76   0.05
3  6.54e+05 3346.12 | 3245.06    0.0   1000     0 |    0.20   0.04  23.39   0.05
3  6.59e+05 3346.12 |  598.90    0.0    212     0 |    0.20   0.05  25.07   0.07
3  6.64e+05 3346.12 |  730.34    0.0    221     0 |    0.21   0.05  26.25   0.07
3  6.68e+05 3346.12 | 3193.28    0.0   1000     0 |    0.20   0.06  26.30   0.07
3  6.73e+05 3346.12 | 3196.86    0.0   1000     0 |    0.21   0.04  25.20   0.05
3  6.77e+05 3346.12 | 3244.45    0.0   1000     0 |    0.20   0.04  23.90   0.05
3  6.83e+05 3346.12 | 3233.50    0.0   1000     0 |    0.21   0.04  23.41   0.05
3  6.88e+05 3346.12 | 3186.67    0.0   1000     0 |    0.21   0.04  23.17   0.05
3  6.92e+05 3346.12 | 3246.38    0.0   1000     0 |    0.22   0.04  23.16   0.05
3  6.97e+05 3346.12 | 3242.70    0.0   1000     0 |    0.21   0.03  22.85   0.05
3  7.02e+05 3346.12 | 3264.54    0.0   1000     0 |    0.21   0.03  22.97   0.05
3  7.07e+05 3346.12 | 3248.13    0.0   1000     0 |    0.22   0.03  23.05   0.05
3  7.11e+05 3346.12 | 3257.00    0.0   1000     0 |    0.20   0.03  22.99   0.04
3  7.16e+05 3346.12 | 3250.57    0.0   1000     0 |    0.19   0.03  23.07   0.05
3  7.21e+05 3346.12 | 3286.31    0.0   1000     0 |    0.21   0.03  22.99   0.05
3  7.25e+05 3346.12 | 3226.14    0.0   1000     0 |    0.21   0.03  23.28   0.05
3  7.30e+05 3346.12 | 3222.68    0.0   1000     0 |    0.22   0.03  23.10   0.05
3  7.34e+05 3346.12 | 3275.78    0.0   1000     0 |    0.22   0.03  22.91   0.04
3  7.39e+05 3346.12 | 3254.49    0.0   1000     0 |    0.21   0.03  22.53   0.04
3  7.44e+05 3346.12 | 3292.13    0.0   1000     0 |    0.22   0.03  22.92   0.05
3  7.49e+05 3346.12 | 3274.65    0.0   1000     0 |    0.22   0.03  22.91   0.05
3  7.53e+05 3346.12 | 3308.70    0.0   1000     0 |    0.22   0.03  23.01   0.04
3  7.59e+05 3346.12 | 3274.37    0.0   1000     0 |    0.22   0.03  23.36   0.04
3  7.63e+05 3346.12 | 3320.37    0.0   1000     0 |    0.22   0.03  23.14   0.05
3  7.68e+05 3346.12 | 3324.92    0.0   1000     0 |    0.23   0.03  23.02   0.05
3  7.73e+05 3346.12 | 3259.66    0.0   1000     0 |    0.21   0.03  23.23   0.05
3  7.77e+05 3346.12 | 3260.96    0.0   1000     0 |    0.22   0.03  22.98   0.05
3  7.81e+05 3346.12 | 3302.02    0.0   1000     0 |    0.22   0.03  23.00   0.05
3  7.86e+05 3346.12 |   28.78    0.0     27     0 |    0.20   0.03  23.68   0.06
3  7.91e+05 3346.12 | 3297.69    0.0   1000     0 |    0.22   0.03  24.46   0.05
3  7.96e+05 3346.12 | 3323.15    0.0   1000     0 |    0.20   0.03  23.52   0.05
3  8.00e+05 3346.12 | 3314.14    0.0   1000     0 |    0.21   0.03  23.75   0.05
3  8.05e+05 3346.12 |  471.67    0.0    181     0 |    0.22   0.04  24.19   0.06
3  8.10e+05 3346.12 | 3219.53    0.0   1000     0 |    0.22   0.04  23.87   0.05
3  8.14e+05 3346.12 | 3254.78    0.0   1000     0 |    0.17   0.03  23.57   0.05
3  8.18e+05 3346.12 | 3174.39    0.0   1000     0 |    0.21   0.04  24.03   0.05
3  8.22e+05 3346.12 | 1044.23    0.0   1000     0 |    0.21   0.04  26.59   0.06
3  8.27e+05 3346.12 | 3258.03    0.0   1000     0 |    0.21   0.04  24.35   0.05
3  8.31e+05 3346.12 |  854.32    0.0    258     0 |    0.07   0.11  25.23   0.07
3  8.36e+05 3346.12 |  964.44    0.0    430     0 |    0.21   0.22  62.71   0.18
3  8.41e+05 3346.12 | 2235.00    0.0   1000     0 |    0.19   0.14  47.49   0.13
3  8.45e+05 3346.12 | 2899.15    0.0   1000     0 |    0.16   4.11  43.89   2.50
3  8.50e+05 3346.12 | 2099.78    0.0   1000     0 |    0.14   0.64  95.06   0.41
3  8.54e+05 3346.12 | 1020.43    0.0   1000     0 |    0.16   0.35  66.75   0.24
3  8.59e+05 3346.12 | 2483.12    0.0   1000     0 |    0.13   0.23  45.98   0.14
3  8.64e+05 3346.12 |  289.95    0.0    115     0 |    0.06   0.37  41.44   0.20
3  8.69e+05 3346.12 | 1026.99    0.0   1000     0 |    0.16   0.13  35.91   0.08
3  8.73e+05 3346.12 | 1426.79    0.0    475     0 |    0.14   0.08  28.36   0.06
3  8.78e+05 3422.72 |
3  8.78e+05 3422.72 | 3422.72    5.7   1000     0 |    0.07   0.06  25.28   0.05
3  8.82e+05 3422.72 | 3372.57    0.0   1000     0 |    0.17   0.04  23.51   0.05
3  8.87e+05 3422.72 | 3331.27    0.0   1000     0 |    0.20   0.04  22.86   0.05
3  8.91e+05 3422.72 | 3309.30    0.0   1000     0 |    0.21   0.04  22.95   0.05
3  8.96e+05 3422.72 | 3277.77    0.0   1000     0 |    0.22   0.04  22.96   0.05
3  9.00e+05 3422.72 | 3320.82    0.0   1000     0 |    0.21   0.04  23.49   0.05
3  9.05e+05 3422.72 | 3326.58    0.0   1000     0 |    0.20   0.04  23.32   0.05
3  9.09e+05 3422.72 |  975.64    0.0   1000     0 |    0.23   0.11  27.91   0.13
3  9.14e+05 3422.72 | 1042.28    0.0   1000     0 |    0.22   0.09  33.67   0.13
3  9.18e+05 3422.72 | 1658.80    0.0   1000     0 |    0.06   0.06  29.10   0.07
3  9.23e+05 3422.72 | 3276.92    0.0   1000     0 |    0.08   0.04  25.73   0.05
3  9.27e+05 3422.72 | 1156.36    0.0   1000     0 |    0.15   0.05  27.06   0.07
3  9.32e+05 3422.72 | 3223.98    0.0   1000     0 |    0.21   0.05  24.56   0.05
3  9.36e+05 3422.72 | 3224.42    0.0   1000     0 |    0.08   0.04  23.76   0.05
3  9.41e+05 3422.72 | 3346.42    0.0   1000     0 |    0.20   0.03  23.10   0.04
3  9.45e+05 3422.72 | 3321.12    0.0   1000     0 |    0.21   0.04  22.89   0.05
3  9.50e+05 3422.72 | 3307.79    0.0   1000     0 |    0.22   0.03  23.41   0.04
3  9.55e+05 3422.72 | 3388.66    0.0   1000     0 |    0.23   0.03  23.29   0.04
3  9.60e+05 3571.28 |
3  9.60e+05 3571.28 | 3571.28  153.4    994    11 |    0.22   0.03  22.61   0.04
3  9.64e+05 3571.28 | 2208.20    0.0    604     0 |    0.22   0.03  22.94   0.04
3  9.69e+05 3571.28 | 3360.15    0.0   1000     0 |    0.22   0.03  23.31   0.04
3  9.73e+05 3571.28 | 3333.36    0.0   1000     0 |    0.23   0.03  22.83   0.04
3  9.78e+05 3571.28 | 3310.99    0.0   1000     0 |    0.23   0.03  23.30   0.05
3  9.82e+05 3571.28 | 3490.81    0.0   1000     0 |    0.23   0.03  23.15   0.05
3  9.87e+05 3571.28 | 3320.36    0.0   1000     0 |    0.23   0.03  22.84   0.04
3  9.91e+05 3571.28 | 3332.88    0.0   1000     0 |    0.23   0.03  22.66   0.04
3  9.96e+05 3571.28 | 3379.00    0.0   1000     0 |    0.23   0.03  23.04   0.04
3  1.00e+06 3571.28 | 3400.09    0.0   1000     0 |    0.22   0.03  22.75   0.04
3  1.01e+06 3571.28 | 3336.84    0.0   1000     0 |    0.23   0.03  23.23   0.04
3  1.01e+06 3571.28 | 3343.45    0.0   1000     0 |    0.22   0.03  22.86   0.05
3  1.02e+06 3571.28 | 3386.21    0.0   1000     0 |    0.22   0.03  23.12   0.04
3  1.02e+06 3571.28 | 3526.69    0.0   1000     0 |    0.22   0.03  23.02   0.04
3  1.03e+06 3571.28 | 3117.71    0.0    827     0 |    0.23   0.03  23.20   0.04
3  1.03e+06 3571.28 | 3156.30    0.0    853     0 |    0.23   0.03  23.28   0.04
3  1.03e+06 3571.28 |  294.54    0.0    113     0 |    0.23   0.17  43.11   0.16
3  1.04e+06 3571.28 | 3199.89    0.0   1000     0 |    0.23   0.09  33.76   0.08
3  1.04e+06 3571.28 | 3196.45    0.0   1000     0 |    0.16   0.05  26.77   0.06
3  1.05e+06 3571.28 | 3350.85    0.0   1000     0 |    0.20   0.04  24.78   0.05
3  1.05e+06 3571.28 | 3282.41    0.0   1000     0 |    0.21   0.04  23.54   0.05
3  1.06e+06 3571.28 | 1015.27    0.0   1000     0 |    0.22   0.11  39.46   0.20
3  1.06e+06 3571.28 | 1011.59    0.0   1000     0 |    0.21   0.09  37.02   0.11
3  1.06e+06 3571.28 | 3270.80    0.0   1000     0 |    0.06   0.05  28.34   0.06
3  1.07e+06 3571.28 | 3296.79    0.0   1000     0 |    0.06   0.04  24.83   0.05
3  1.07e+06 3571.28 | 1010.46    0.0   1000     0 |    0.20   0.16  42.20   0.20
3  1.08e+06 3571.28 | 1031.59    0.0   1000     0 |    0.21   0.17  49.98   0.19
3  1.08e+06 3571.28 | 3305.42    0.0   1000     0 |    0.06   0.09  34.40   0.08
3  1.09e+06 3571.28 | 3317.12    0.0   1000     0 |    0.06   0.05  27.39   0.06
3  1.09e+06 3571.28 | 3300.44    0.0   1000     0 |    0.20   0.04  24.40   0.05
3  1.10e+06 3571.28 | 3374.11    0.0   1000     0 |    0.20   0.04  23.73   0.05
3  1.10e+06 3571.28 | 3356.14    0.0   1000     0 |    0.20   0.03  22.98   0.04
3  1.11e+06 3571.28 | 3429.12    0.0   1000     0 |    0.21   0.03  23.23   0.04
3  1.11e+06 3571.28 | 3546.16    0.0   1000     0 |    0.22   0.03  23.31   0.04
3  1.12e+06 3571.28 | 3353.44    0.0   1000     0 |    0.23   0.03  22.94   0.04
3  1.12e+06 3571.28 | 3330.81    0.0   1000     0 |    0.23   0.03  22.57   0.04
3  1.13e+06 3571.28 | 3539.38    0.0   1000     0 |    0.23   0.03  22.59   0.04
3  1.13e+06 3571.28 | 3367.10    0.0   1000     0 |    0.23   0.03  22.98   0.04
3  1.14e+06 3571.28 | 3400.75    0.0   1000     0 |    0.23   0.03  22.64   0.04
3  1.14e+06 3571.28 | 3353.39    0.0   1000     0 |    0.23   0.03  22.98   0.04
3  1.15e+06 3571.28 | 3350.79    0.0   1000     0 |    0.23   0.03  23.00   0.04
3  1.15e+06 3571.28 | 3409.38    0.0   1000     0 |    0.23   0.03  23.08   0.04
3  1.15e+06 3571.28 | 3322.69    0.0   1000     0 |    0.22   0.03  23.28   0.05
3  1.16e+06 3571.28 | 3480.63    0.0   1000     0 |    0.22   0.03  23.43   0.04
3  1.16e+06 3571.28 | 3471.95    0.0   1000     0 |    0.23   0.03  22.56   0.04
3  1.17e+06 3571.28 | 3420.43    0.0   1000     0 |    0.23   0.03  23.00   0.04
3  1.17e+06 3571.28 | 3451.60    0.0   1000     0 |    0.23   0.03  22.84   0.04
3  1.18e+06 3571.28 | 3480.65    0.0   1000     0 |    0.23   0.03  23.21   0.04
3  1.18e+06 3571.28 | 3299.38    0.0   1000     0 |    0.23   0.03  23.08   0.04
3  1.19e+06 3571.28 | 3424.38    0.0   1000     0 |    0.23   0.03  22.86   0.04
3  1.19e+06 3571.28 | 3479.01    0.0   1000     0 |    0.23   0.03  23.10   0.04
3  1.20e+06 3571.28 | 3445.37    0.0   1000     0 |    0.23   0.03  22.98   0.04
3  1.20e+06 3571.28 | 3379.36    0.0   1000     0 |    0.24   0.03  22.98   0.04
3  1.21e+06 3571.28 | 3406.81    0.0   1000     0 |    0.23   0.03  23.53   0.04
3  1.21e+06 3571.28 | 3393.73    0.0   1000     0 |    0.23   0.03  23.01   0.04
3  1.21e+06 3571.28 | 3484.27    0.0   1000     0 |    0.23   0.02  23.16   0.04
3  1.22e+06 3571.28 | 1042.91    0.0   1000     0 |    0.23   0.03  25.02   0.05
3  1.22e+06 3571.28 | 3328.04    0.0   1000     0 |    0.23   0.03  23.60   0.04
3  1.23e+06 3571.28 |  431.63    0.0    161     0 |    0.07   0.03  23.69   0.04
3  1.23e+06 3571.28 | 3322.50    0.0   1000     0 |    0.23   0.03  22.57   0.04
3  1.24e+06 3571.28 | 3283.49    0.0   1000     0 |    0.22   0.03  23.31   0.04
3  1.24e+06 3571.28 | 1090.55    0.0   1000     0 |    0.22   0.05  27.79   0.10
3  1.25e+06 3571.28 | 3300.28    0.0   1000     0 |    0.23   0.03  25.05   0.05
3  1.25e+06 3571.28 | 1051.65    0.0   1000     0 |    0.07   0.04  25.91   0.06
3  1.25e+06 3571.28 | 3308.98    0.0   1000     0 |    0.21   0.03  24.53   0.05
3  1.26e+06 3571.28 | 3304.30    0.0   1000     0 |    0.13   0.03  24.05   0.04
3  1.26e+06 3571.28 | 3509.68    0.0   1000     0 |    0.22   0.03  23.51   0.04
3  1.27e+06 3571.28 | 1866.43    0.0    528     0 |    0.21   0.03  23.08   0.04
3  1.27e+06 3571.28 | 3329.80    0.0   1000     0 |    0.23   0.03  23.43   0.05
3  1.28e+06 3571.28 | 3422.68  326.1    944    98 |    0.23   0.03  23.62   0.04
3  1.28e+06 3571.28 | 1058.56    0.0    450     0 |    0.22   0.06  28.74   0.10
3  1.29e+06 3571.28 | 3205.49    0.0   1000     0 |    0.23   0.04  25.44   0.05
3  1.29e+06 3571.28 | 3327.98    0.0   1000     0 |    0.17   0.03  23.57   0.05
3  1.29e+06 3571.28 | 3366.74    0.0   1000     0 |    0.21   0.03  23.18   0.05
3  1.30e+06 3571.28 | 3394.69    0.0   1000     0 |    0.23   0.03  23.19   0.04
3  1.30e+06 3571.28 | 3355.44    0.0   1000     0 |    0.22   0.03  22.85   0.04
3  1.31e+06 3571.28 | 3503.05    0.0   1000     0 |    0.22   0.03  23.22   0.04
3  1.31e+06 3571.28 | 3368.96    0.0   1000     0 |    0.23   0.03  23.36   0.05
3  1.32e+06 3571.28 | 3515.65    0.0    959     0 |    0.23   0.03  23.59   0.04
| UsedTime:   56106 | SavedDir: ./Hopper-v2_ReliableSAC_3


        """
    elif env_name == 'Hopper-v2':
        env_func = gym.make
        env_args = {
            'env_num': 1,
            'env_name': 'Hopper-v2',
            'max_step': 1000,
            'state_dim': 11,
            'action_dim': 3,
            'if_discrete': False,
            'target_return': 3800.,
        }
        args = Arguments(agent, env_func=env_func, env_args=env_args)
        args.eval_times = 2 ** 2
        args.reward_scale = 2 ** -4

        args.target_step = args.max_step * 2
        args.worker_num = 2

        args.net_dim = 2 ** 8
        args.num_layer = 3
        args.batch_size = int(args.net_dim * 2)
        args.repeat_times = 2 ** 4
        args.gamma = 0.993  # todo

        args.if_allow_break = False
        args.break_step = int(8e6)

        """
################################################################################
ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
6  4.05e+03   34.11 |
6  4.05e+03   34.11 |   34.11    0.4     38     0 |    0.05   0.80   0.84   0.33
6  1.17e+05   62.90 |
6  1.17e+05   62.90 |   62.90    0.7     38     0 |    0.11   0.33  14.94   0.28
6  1.96e+05  313.13 |
6  1.96e+05  313.13 |  313.13    0.8    140     0 |    0.13   0.59  30.46   0.20
6  2.56e+05  313.13 |  288.39    0.0    125     0 |    0.14   0.59  36.48   0.15
6  3.04e+05  313.13 |  268.59    0.0    117     0 |    0.14   0.49  32.35   0.14
6  3.44e+05  313.13 |    7.47    0.0     33     0 |    0.12   0.36  25.97   0.13
6  3.83e+05 1038.82 |
6  3.83e+05 1038.82 | 1038.82    0.2   1000     0 |    0.07   0.32  29.84   0.17
6  4.22e+05 1038.82 | 1028.53    0.0   1000     0 |    0.08   0.32  31.94   0.18
6  4.57e+05 1038.82 | 1024.58    0.0   1000     0 |    0.09   0.31  29.22   0.14
6  4.89e+05 1054.37 |
6  4.89e+05 1054.37 | 1054.37    6.0   1000     0 |    0.07   0.30  32.58   0.15
6  5.18e+05 1054.37 |  345.63    0.0    152     0 |    0.14   0.32  33.13   0.14
6  5.44e+05 1054.37 |  181.06    0.0    110     0 |    0.10   0.28  34.14   0.14
6  5.71e+05 1054.37 |  602.82    0.0    215     0 |    0.15   0.39  31.36   0.12
6  5.95e+05 1054.37 | 1035.55    0.0   1000     0 |    0.10   0.40  33.95   0.15
6  6.19e+05 1054.37 |  295.51    0.0    126     0 |    0.15   0.51  31.59   0.14
6  6.44e+05 1054.37 |   64.50    0.0     43     0 |    0.07   0.46  33.29   0.17
6  6.66e+05 1054.37 |  643.23    0.0    284     0 |    0.14   0.37  34.44   0.15
6  6.89e+05 1054.37 |  501.73    0.0    173     0 |    0.11   0.32  35.48   0.16
6  7.08e+05 1054.37 |  388.54    0.0    214     0 |    0.13   0.27  34.18   0.13
6  7.29e+05 1054.37 |  458.66    0.0    304     0 |    0.08   0.26  31.83   0.14
6  7.48e+05 1054.37 |  254.32    0.0    146     0 |    0.12   0.22  29.52   0.11
6  7.68e+05 1054.37 | 1028.87    0.0   1000     0 |    0.08   0.19  26.58   0.10
6  7.86e+05 1054.37 |  240.32    0.0    196     0 |    0.08   0.16  25.03   0.10
6  8.03e+05 1054.37 |  108.17    0.0     73     0 |    0.08   0.15  23.71   0.10
6  8.22e+05 1054.37 | 1029.23    0.0   1000     0 |    0.08   0.15  22.94   0.08
6  8.43e+05 1054.37 | 1039.32    0.0   1000     0 |    0.07   0.13  22.86   0.08
6  8.64e+05 1054.37 | 1036.23    0.0   1000     0 |    0.07   0.12  21.10   0.07
6  8.83e+05 2471.51 |
6  8.83e+05 2471.51 | 2471.51    2.0   1000     0 |    0.14   0.11  20.02   0.07
6  8.99e+05 2471.51 |  271.53    0.0    136     0 |    0.14   0.10  19.55   0.06
6  9.16e+05 2471.51 |  788.17    0.0    328     0 |    0.15   0.11  19.51   0.07
6  9.32e+05 2471.51 |  305.98    0.0    143     0 |    0.14   0.10  19.81   0.07
6  9.49e+05 2471.51 |  213.25    0.0    115     0 |    0.16   0.11  19.30   0.06
6  9.65e+05 2471.51 |  212.62    0.0    116     0 |    0.17   0.09  18.55   0.06
6  9.80e+05 2471.51 |  231.38    0.0    121     0 |    0.16   0.09  18.55   0.06
6  9.96e+05 2816.96 |
6  9.96e+05 2816.96 | 2816.96    1.5   1000     0 |    0.18   0.09  18.83   0.06
6  1.01e+06 2816.96 | 2526.01    0.0   1000     0 |    0.19   0.08  19.41   0.06
6  1.03e+06 2816.96 | 2502.35    0.0   1000     0 |    0.19   0.09  19.39   0.06
6  1.04e+06 2816.96 | 2792.51    0.0   1000     0 |    0.19   0.09  19.62   0.06
6  1.06e+06 2816.96 | 2807.12   18.1   1000     0 |    0.18   0.09  19.92   0.06
6  1.07e+06 2816.96 | 1274.49    0.0    452     0 |    0.20   0.08  19.97   0.06
6  1.08e+06 2827.08 |
6  1.08e+06 2827.08 | 2827.08    8.4   1000     0 |    0.19   0.08  20.43   0.06
6  1.10e+06 2827.08 | 1578.28    0.0    569     0 |    0.19   0.08  20.81   0.06
6  1.11e+06 2871.91 |
6  1.11e+06 2871.91 | 2871.91    0.9   1000     0 |    0.20   0.08  20.87   0.06
6  1.13e+06 2871.91 | 2688.35    0.0   1000     0 |    0.19   0.08  21.17   0.06
6  1.14e+06 2871.91 | 2681.34    0.0   1000     0 |    0.18   0.08  21.20   0.06
6  1.16e+06 2871.91 | 2810.03    0.0   1000     0 |    0.18   0.08  22.26   0.06
6  1.18e+06 2871.91 | 1660.03    0.0    584     0 |    0.18   0.08  21.62   0.06
6  1.19e+06 2871.91 | 2639.25    0.0   1000     0 |    0.18   0.07  21.81   0.06
6  1.21e+06 2871.91 |  939.60    0.0    312     0 |    0.18   0.07  21.66   0.06
6  1.22e+06 2871.91 | 2716.22    0.0   1000     0 |    0.18   0.08  21.85   0.06
6  1.24e+06 2871.91 | 2697.94    0.0   1000     0 |    0.19   0.07  21.58   0.06
6  1.25e+06 2871.91 | 2739.97    0.0   1000     0 |    0.19   0.07  22.32   0.06
6  1.26e+06 2871.91 | 2726.54    0.0   1000     0 |    0.19   0.07  22.24   0.06
6  1.27e+06 2871.91 | 2596.29    0.0   1000     0 |    0.20   0.07  22.64   0.06
6  1.29e+06 2871.91 | 2695.79    0.0   1000     0 |    0.19   0.07  22.49   0.06
6  1.30e+06 2871.91 | 2513.97    0.0   1000     0 |    0.18   0.07  22.17   0.06
6  1.31e+06 2871.91 | 2653.65    0.0   1000     0 |    0.20   0.06  21.88   0.06
6  1.32e+06 2871.91 | 2545.89    0.0   1000     0 |    0.19   0.06  22.11   0.06
6  1.34e+06 2871.91 | 2627.76    0.0   1000     0 |    0.17   0.06  22.41   0.06
6  1.35e+06 2871.91 | 2512.66    0.0   1000     0 |    0.18   0.06  21.69   0.06
6  1.36e+06 2871.91 | 2614.05    0.0   1000     0 |    0.19   0.06  22.41   0.06
6  1.37e+06 2871.91 | 2511.88    0.0   1000     0 |    0.18   0.06  22.86   0.06
6  1.38e+06 2871.91 | 2511.73    0.0   1000     0 |    0.18   0.06  22.40   0.06
6  1.39e+06 2871.91 | 2576.39    0.0   1000     0 |    0.18   0.06  22.36   0.06
6  1.40e+06 2871.91 | 2593.05    0.0   1000     0 |    0.18   0.06  22.56   0.06
6  1.42e+06 2871.91 | 2612.13    0.0   1000     0 |    0.19   0.06  22.38   0.06
6  1.43e+06 2871.91 | 2709.74    0.0   1000     0 |    0.18   0.06  22.08   0.06
6  1.44e+06 2871.91 | 2762.59    0.0   1000     0 |    0.18   0.06  22.09   0.06
6  1.45e+06 2871.91 | 2746.69    0.0   1000     0 |    0.18   0.06  21.86   0.06
6  1.46e+06 2871.91 | 2637.90    0.0   1000     0 |    0.18   0.06  22.09   0.06
6  1.48e+06 2871.91 | 2712.50    0.0   1000     0 |    0.18   0.05  22.10   0.06
6  1.49e+06 2871.91 | 2703.42    0.0   1000     0 |    0.18   0.06  22.62   0.06
6  1.50e+06 2871.91 | 2767.33    0.0   1000     0 |    0.17   0.05  21.92   0.06
6  1.51e+06 2919.71 |
6  1.51e+06 2919.71 | 2919.71    4.6   1000     0 |    0.18   0.06  22.39   0.06
6  1.52e+06 2919.71 | 2878.41    0.0   1000     0 |    0.18   0.06  22.34   0.06
6  1.54e+06 2919.71 | 2706.40    0.0   1000     0 |    0.19   0.06  22.33   0.05
6  1.55e+06 2919.71 | 2766.00    0.0   1000     0 |    0.19   0.05  22.25   0.05
6  1.56e+06 2919.71 | 2686.77    0.0   1000     0 |    0.19   0.05  22.11   0.05
6  1.57e+06 2919.71 | 2806.31    0.0   1000     0 |    0.19   0.05  22.06   0.05
6  1.58e+06 2919.71 | 2806.25    0.0   1000     0 |    0.18   0.05  21.52   0.05
6  1.59e+06 2919.71 | 2851.65    0.0   1000     0 |    0.19   0.05  21.15   0.05
6  1.60e+06 2919.71 | 2799.51    0.0   1000     0 |    0.18   0.05  21.29   0.05
6  1.61e+06 2919.71 | 2853.32    0.0   1000     0 |    0.18   0.05  21.43   0.05
6  1.62e+06 2919.71 | 2776.67    0.0   1000     0 |    0.18   0.05  21.82   0.05
6  1.63e+06 2919.71 | 2846.76    0.0   1000     0 |    0.18   0.05  21.45   0.05
6  1.64e+06 2919.71 | 2792.80    0.0   1000     0 |    0.19   0.05  21.70   0.05
6  1.65e+06 2919.71 | 2827.58    0.0   1000     0 |    0.19   0.05  21.97   0.05
6  1.66e+06 2919.71 | 2784.43    0.0   1000     0 |    0.19   0.05  21.33   0.05
6  1.67e+06 2919.71 | 2737.03    0.0   1000     0 |    0.19   0.04  21.52   0.05
6  1.68e+06 2919.71 | 2812.33    0.0   1000     0 |    0.18   0.04  21.24   0.05
6  1.69e+06 2919.71 | 2788.36    0.0   1000     0 |    0.18   0.05  21.54   0.05
6  1.70e+06 2919.71 | 2786.58    0.0   1000     0 |    0.19   0.04  21.31   0.05
6  1.71e+06 2919.71 | 2742.50    0.0   1000     0 |    0.18   0.04  21.51   0.05
6  1.72e+06 2919.71 | 2720.79    0.0   1000     0 |    0.19   0.05  21.70   0.05
6  1.73e+06 2919.71 | 2786.67    0.0   1000     0 |    0.18   0.04  21.73   0.05
6  1.74e+06 2919.71 | 2843.51    0.0   1000     0 |    0.19   0.05  21.66   0.05
6  1.75e+06 2919.71 | 2808.91    0.0   1000     0 |    0.19   0.05  21.34   0.05
6  1.76e+06 2919.71 | 2796.23    0.0   1000     0 |    0.19   0.04  21.98   0.05
6  1.78e+06 2919.71 | 2887.21    0.0   1000     0 |    0.19   0.05  21.69   0.05
6  1.79e+06 2919.71 | 2779.83    0.0   1000     0 |    0.18   0.05  21.79   0.05
6  1.80e+06 2919.71 | 2910.02    0.0   1000     0 |    0.19   0.05  21.81   0.05
6  1.81e+06 2978.96 |
6  1.81e+06 2978.96 | 2978.96    0.6   1000     0 |    0.19   0.04  22.15   0.05
6  1.82e+06 2990.69 |
6  1.82e+06 2990.69 | 2990.69    4.2   1000     0 |    0.20   0.04  21.37   0.05
6  1.83e+06 2990.69 | 2946.43    0.0   1000     0 |    0.19   0.05  22.07   0.05
6  1.84e+06 2990.69 | 2840.06    0.0   1000     0 |    0.19   0.04  22.10   0.05
6  1.85e+06 2990.69 | 2939.27    0.0   1000     0 |    0.19   0.05  21.63   0.05
6  1.86e+06 2990.69 | 2851.73    0.0   1000     0 |    0.19   0.05  22.16   0.05
6  1.87e+06 2990.69 | 2902.68    0.0   1000     0 |    0.19   0.05  22.01   0.05
6  1.88e+06 2990.69 | 2891.08    0.0   1000     0 |    0.19   0.05  22.01   0.05
6  1.89e+06 2990.69 | 2874.25    0.0   1000     0 |    0.19   0.04  21.42   0.05
6  1.90e+06 2990.69 | 2858.17    0.0   1000     0 |    0.19   0.04  22.03   0.05
6  1.91e+06 2990.69 | 2885.14    0.0   1000     0 |    0.19   0.04  21.90   0.05
6  1.92e+06 2990.69 | 2835.17    0.0   1000     0 |    0.19   0.04  21.43   0.05
6  1.93e+06 2990.69 | 2842.48    0.0   1000     0 |    0.18   0.04  21.61   0.05
6  1.94e+06 2990.69 | 2881.02    0.0   1000     0 |    0.19   0.04  21.86   0.05
6  1.95e+06 2990.69 | 2960.67    0.0   1000     0 |    0.18   0.04  22.14   0.05
6  1.96e+06 2990.69 | 2947.27    0.0   1000     0 |    0.19   0.04  22.05   0.05
6  1.97e+06 2990.69 | 2901.40    0.0   1000     0 |    0.20   0.04  21.81   0.05
6  1.98e+06 2990.69 | 2927.26    0.0   1000     0 |    0.19   0.04  22.16   0.05
6  1.99e+06 2990.69 | 2960.03    0.0   1000     0 |    0.20   0.04  21.71   0.04
6  2.00e+06 2993.62 |
6  2.00e+06 2993.62 | 2993.62    2.0   1000     0 |    0.19   0.04  21.76   0.04
6  2.01e+06 2993.62 | 2903.15    0.0   1000     0 |    0.20   0.04  21.60   0.04
6  2.02e+06 2993.62 | 2811.17    0.0   1000     0 |    0.19   0.04  21.50   0.04
6  2.03e+06 2993.62 | 2878.24    0.0   1000     0 |    0.19   0.04  21.86   0.04
6  2.04e+06 2993.62 | 2916.72    0.0   1000     0 |    0.19   0.04  21.96   0.04
6  2.05e+06 2993.62 | 2891.63    0.0   1000     0 |    0.19   0.04  21.99   0.04
6  2.06e+06 2993.62 | 2965.31    0.0   1000     0 |    0.19   0.04  21.86   0.04
6  2.07e+06 2993.62 | 2910.02    0.0   1000     0 |    0.19   0.04  21.89   0.04
6  2.08e+06 2993.62 | 2942.60    0.0   1000     0 |    0.19   0.04  21.88   0.04
6  2.09e+06 2999.49 |
6  2.09e+06 2999.49 | 2999.49    1.1   1000     0 |    0.19   0.04  21.68   0.04
6  2.09e+06 2999.49 | 2967.17    0.0   1000     0 |    0.19   0.04  22.16   0.04
6  2.10e+06 2999.49 | 2937.72    0.0   1000     0 |    0.19   0.04  21.36   0.04
6  2.10e+06 2999.49 | 2937.72    0.0   1000     0 |    0.19   0.04  21.36   0.04
6  2.11e+06 2999.49 | 2947.81    0.0   1000     0 |    0.20   0.04  20.62   0.04
6  2.12e+06 2999.49 | 2928.85    0.0   1000     0 |    0.20   0.04  21.45   0.04
6  2.12e+06 3005.13 |
6  2.12e+06 3005.13 | 3005.13    0.9   1000     0 |    0.19   0.04  21.66   0.04
6  2.13e+06 3005.13 | 2968.58    0.0   1000     0 |    0.19   0.04  21.54   0.04
6  2.14e+06 3007.51 |
6  2.14e+06 3007.51 | 3007.51    0.5   1000     0 |    0.20   0.04  21.40   0.04
6  2.15e+06 3007.51 | 2958.19    0.0   1000     0 |    0.20   0.04  21.94   0.04
6  2.16e+06 3059.47 |
6  2.16e+06 3059.47 | 3059.47    1.8   1000     0 |    0.19   0.04  21.89   0.04
6  2.17e+06 3059.47 | 3039.30    0.0   1000     0 |    0.19   0.04  22.13   0.04
6  2.18e+06 3079.37 |
6  2.18e+06 3079.37 | 3079.37    0.4   1000     0 |    0.20   0.04  22.02   0.04
6  2.18e+06 3079.37 | 3036.29    0.0   1000     0 |    0.20   0.03  21.91   0.04
6  2.20e+06 3079.37 | 3045.43    0.0   1000     0 |    0.20   0.04  21.75   0.04
6  2.21e+06 3079.37 | 3078.21    0.0   1000     0 |    0.20   0.04  21.72   0.04
6  2.21e+06 3079.37 | 3030.94    0.0   1000     0 |    0.19   0.03  21.58   0.04
6  2.22e+06 3079.37 | 3034.21    0.0   1000     0 |    0.19   0.03  21.51   0.04
6  2.23e+06 3079.37 | 3065.70    0.0   1000     0 |    0.19   0.03  21.91   0.04
6  2.23e+06 3079.37 | 3027.77    0.0   1000     0 |    0.19   0.03  21.56   0.04
6  2.24e+06 3079.37 | 3050.05    0.0   1000     0 |    0.19   0.03  21.51   0.04
6  2.25e+06 3079.37 | 3055.75    0.0   1000     0 |    0.19   0.03  21.59   0.04
6  2.26e+06 3079.37 | 3029.73    0.0   1000     0 |    0.20   0.03  21.99   0.04
6  2.26e+06 3079.37 | 3075.19    0.0   1000     0 |    0.20   0.03  21.55   0.04
6  2.27e+06 3079.37 | 3020.89    0.0   1000     0 |    0.19   0.03  21.20   0.04
6  2.28e+06 3079.37 | 3069.39    0.0   1000     0 |    0.19   0.03  21.37   0.04
6  2.28e+06 3079.37 | 3065.39    0.0   1000     0 |    0.19   0.03  21.35   0.04
6  2.29e+06 3079.37 | 3073.13    0.0   1000     0 |    0.19   0.03  22.03   0.04
6  2.30e+06 3079.37 | 3050.63    0.0   1000     0 |    0.19   0.03  21.24   0.04
6  2.30e+06 3079.37 | 3078.68    0.0   1000     0 |    0.20   0.03  21.41   0.04
6  2.31e+06 3079.37 | 3036.35    0.0   1000     0 |    0.19   0.03  21.59   0.04
6  2.32e+06 3079.37 |  414.22    0.0    278     0 |    0.01   0.30  49.33   0.06
6  2.33e+06 3079.37 | 1001.89    0.0   1000     0 |    0.06   0.17  42.27   0.08
6  2.33e+06 3079.37 | 1015.33    0.0   1000     0 |    0.06   0.13  37.14   0.11
6  2.34e+06 3079.37 | 1011.02    0.0   1000     0 |    0.06   0.10  34.35   0.10
6  2.35e+06 3079.37 | 2813.60    0.0   1000     0 |    0.07   0.12  30.99   0.08
6  2.35e+06 3079.37 |  505.06    0.0    195     0 |    0.18   0.10  29.30   0.08
6  2.36e+06 3079.37 |  333.13    0.0    125     0 |    0.17   0.10  29.47   0.08
6  2.37e+06 3079.37 |  628.32    0.0    262     0 |    0.15   0.08  29.50   0.08
6  2.37e+06 3079.37 | 1799.29    0.0    623     0 |    0.15   0.08  27.85   0.07
6  2.38e+06 3079.37 | 3069.86   10.1   1000     0 |    0.16   0.08  26.93   0.07
6  2.39e+06 3079.37 |  939.58    0.0    278     0 |    0.18   0.07  25.53   0.06
6  2.39e+06 3079.37 | 3046.63    0.0   1000     0 |    0.20   0.08  25.15   0.06
6  2.40e+06 3187.65 |
6  2.40e+06 3187.65 | 3187.65    3.1   1000     0 |    0.19   0.07  24.49   0.05

        
        """
    elif env_name == 'Humanoid-v3.9400.best':
        from elegantrl.envs.CustomGymEnv import HumanoidEnv
        env_func = HumanoidEnv
        env_args = {
            'env_num': 1,
            'env_name': 'Humanoid-v3',
            'max_step': 1000,
            'state_dim': 376,
            'action_dim': 17,
            'if_discrete': False,
            'target_return': 3000.,
        }
        args = Arguments(agent, env_func=env_func, env_args=env_args)

        args.eval_times = 2 ** 2
        args.reward_scale = 2 ** -2
        args.max_memo = 2 ** 21
        args.learning_rate = 2 ** -14
        args.lambda_a_log_std = 2 ** -6

        args.target_step = args.max_step
        args.worker_num = 4

        args.net_dim = 2 ** 9
        args.batch_size = args.net_dim // 2
        args.num_layer = 3
        args.repeat_times = 2 ** 0
        args.gamma = 0.96
        args.if_act_target = False
        import numpy as np
        args.target_entropy = np.log(env_args['action_dim'])

        args.if_allow_break = False
        args.break_step = int(4e6)

        """
| Arguments Remove cwd: ./Humanoid-v3_ReliableSAC_3
################################################################################
ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
3  8.09e+03  176.45 |
3  8.09e+03  176.45 |  176.45   29.6     34     6 |    1.19   0.69   2.81   0.06
3  1.47e+05  216.06 |
3  1.47e+05  216.06 |  216.06   12.1     41     2 |    1.27   0.79  32.29   0.03
3  2.25e+05  289.98 |
3  2.25e+05  289.98 |  289.98   13.7     56     2 |    1.34   0.68  31.95   0.01
3  2.83e+05  316.69 |
3  2.83e+05  316.69 |  316.69   17.4     58     3 |    1.34   0.45  27.33   0.01
3  3.28e+05  316.69 |  287.80    0.0     57     0 |    1.28   0.33  24.88   0.01
3  3.70e+05  379.81 |
3  3.70e+05  379.81 |  379.81   36.3     71     7 |    1.31   0.29  24.75   0.01
3  4.07e+05  379.81 |  209.17    0.0     41     0 |    1.35   0.26  25.31   0.01
3  4.40e+05  379.81 |  356.76    0.0     67     0 |    1.36   0.24  25.37   0.01
3  4.74e+05  379.81 |  374.84   36.5     67     6 |    1.34   0.22  25.21   0.01
3  5.03e+05  525.94 |
3  5.03e+05  525.94 |  525.94   41.6    100    11 |    1.36   0.21  25.63   0.01
3  5.32e+05  525.94 |  497.22    0.0     89     0 |    1.36   0.20  25.51   0.01
3  5.62e+05  525.94 |  334.79    0.0     59     0 |    1.37   0.20  26.24   0.01
3  5.87e+05  525.94 |  428.25    0.0     76     0 |    1.39   0.19  25.51   0.01
3  6.12e+05  525.94 |  423.34    0.0     74     0 |    1.37   0.18  26.50   0.01
3  6.37e+05  525.94 |  479.20    0.0     84     0 |    1.40   0.18  26.26   0.01
3  6.62e+05  551.04 |
3  6.62e+05  551.04 |  551.04   83.8     98    16 |    1.39   0.18  26.49   0.01
3  6.83e+05  551.04 |  397.43    0.0     70     0 |    1.36   0.18  27.22   0.01
3  7.04e+05  745.68 |
3  7.04e+05  745.68 |  745.68  106.0    134    15 |    1.35   0.17  26.33   0.01
3  7.25e+05  745.68 |  509.76    0.0     89     0 |    1.42   0.17  26.93   0.01
3  7.47e+05  745.68 |  376.85    0.0     70     0 |    1.41   0.18  28.16   0.01
3  7.69e+05  745.68 |  580.24  127.2    104    23 |    1.38   0.18  27.97   0.01
3  7.91e+05  745.68 |  491.18    0.0     85     0 |    1.40   0.18  27.98   0.01
3  8.12e+05  745.68 |  492.97    0.0     87     0 |    1.41   0.18  29.00   0.01
3  8.34e+05  801.10 |
3  8.34e+05  801.10 |  801.10  244.9    146    39 |    1.43   0.18  28.95   0.01
3  8.51e+05  801.10 |  522.10    0.0     93     0 |    1.42   0.18  28.90   0.01
3  8.69e+05  801.10 |  620.43    0.0    105     0 |    1.44   0.18  29.90   0.01
3  8.85e+05  829.16 |
3  8.85e+05  829.16 |  829.16  109.8    152    30 |    1.42   0.17  29.57   0.01
3  9.03e+05  829.16 |  733.06  127.1    135    32 |    1.47   0.17  30.11   0.01
3  9.20e+05 1018.17 |
3  9.20e+05 1018.17 | 1018.17  329.5    174    53 |    1.43   0.17  30.46   0.01
3  9.37e+05 1018.17 |  739.87    0.0    135     0 |    1.45   0.18  28.93   0.01
3  9.55e+05 1018.17 |  744.00    0.0    129     0 |    1.46   0.18  31.50   0.01
3  9.73e+05 1208.41 |
3  9.73e+05 1208.41 | 1208.41  378.3    204    57 |    1.48   0.17  29.71   0.01
3  9.90e+05 1208.41 |  647.54    0.0    117     0 |    1.45   0.18  30.89   0.01
3  1.01e+06 1208.41 |  929.52    0.0    163     0 |    1.47   0.18  30.55   0.01
3  1.02e+06 2111.40 |
3  1.02e+06 2111.40 | 2111.40  513.4    344    77 |    1.50   0.18  30.26   0.01
3  1.04e+06 2961.21 |
3  1.04e+06 2961.21 | 2961.21 1907.8    467   300 |    1.49   0.17  31.73   0.01
3  1.06e+06 2961.21 | 1923.17    0.0    307     0 |    1.51   0.18  31.26   0.01
3  1.08e+06 2961.21 |  665.96    0.0    115     0 |    1.51   0.17  32.32   0.01
3  1.10e+06 2961.21 | 2009.69    0.0    308     0 |    1.56   0.17  32.11   0.01
3  1.11e+06 2961.21 | 1253.18    0.0    193     0 |    1.54   0.17  31.96   0.02
3  1.13e+06 2961.21 |  822.65    0.0    125     0 |    1.56   0.17  32.47   0.02
3  1.15e+06 2961.21 |  808.09    0.0    126     0 |    1.58   0.18  32.09   0.02
3  1.16e+06 2961.21 | 1995.99    0.0    317     0 |    1.58   0.17  32.54   0.02
3  1.17e+06 2961.21 |  804.38    0.0    128     0 |    1.59   0.18  33.59   0.02
3  1.19e+06 2961.21 | 1058.70    0.0    169     0 |    1.58   0.18  32.22   0.02
3  1.20e+06 2961.21 |  954.06    0.0    155     0 |    1.61   0.18  33.44   0.02
3  1.21e+06 2961.21 | 1245.58    0.0    195     0 |    1.62   0.18  32.46   0.02
3  1.23e+06 2961.21 | 2780.23    0.0    415     0 |    1.62   0.18  33.10   0.02
3  1.24e+06 4273.43 |
3  1.24e+06 4273.43 | 4273.43 1977.1    643   289 |    1.63   0.17  33.10   0.02
3  1.26e+06 4273.43 |  304.42    0.0     53     0 |    1.63   0.18  33.53   0.02
3  1.27e+06 4273.43 | 4017.09 2192.0    596   319 |    1.65   0.18  33.96   0.02
3  1.29e+06 4273.43 | 4190.22 2639.0    604   374 |    1.65   0.18  34.71   0.02
3  1.30e+06 4273.43 | 1137.88    0.0    185     0 |    1.68   0.18  33.78   0.02
3  1.31e+06 4273.43 |  977.47    0.0    143     0 |    1.65   0.18  34.35   0.02
3  1.33e+06 4969.29 |
3  1.33e+06 4969.29 | 4969.29 2784.2    712   393 |    1.72   0.18  34.51   0.02
3  1.34e+06 4969.29 | 4573.45    0.0    634     0 |    1.71   0.18  35.18   0.02
3  1.36e+06 4969.29 | 4896.34    0.0    663     0 |    1.76   0.18  35.41   0.02
3  1.37e+06 4969.29 | 1326.51    0.0    194     0 |    1.67   0.18  34.70   0.02
3  1.39e+06 4969.29 | 1680.32    0.0    238     0 |    1.80   0.18  35.17   0.02
3  1.40e+06 4969.29 | 3260.77    0.0    451     0 |    1.74   0.18  34.56   0.02
3  1.42e+06 4969.29 | 1140.55    0.0    162     0 |    1.80   0.18  34.71   0.02
3  1.43e+06 4969.29 | 2484.06 2403.7    340   320 |    1.79   0.18  35.37   0.02
3  1.45e+06 4969.29 | 2997.63    0.0    422     0 |    1.81   0.19  35.35   0.02
3  1.46e+06 4969.29 | 4737.24 1827.3    641   242 |    1.85   0.18  35.85   0.02
3  1.48e+06 4969.29 | 1047.71    0.0    148     0 |    1.83   0.19  35.50   0.02
3  1.49e+06 4969.29 |  173.17    0.0     33     0 |    1.84   0.19  36.82   0.02
3  1.51e+06 4969.29 | 3679.41    0.0    497     0 |    1.86   0.19  36.89   0.02
3  1.52e+06 4969.29 | 4962.11 2734.9    652   348 |    1.88   0.19  37.78   0.02
3  1.53e+06 4969.29 | 2338.43    0.0    305     0 |    1.86   0.19  36.17   0.02
3  1.55e+06 4969.29 | 2442.01    0.0    309     0 |    1.81   0.19  36.69   0.02
3  1.56e+06 4969.29 | 4154.86 2039.7    522   250 |    1.85   0.19  37.91   0.02
3  1.58e+06 4969.29 | 4003.55    0.0    502     0 |    1.84   0.18  37.31   0.02
3  1.59e+06 4969.29 | 3509.02    0.0    445     0 |    1.88   0.19  37.35   0.02
3  1.61e+06 4969.29 | 3969.27 2935.2    520   379 |    1.86   0.19  36.67   0.02
3  1.62e+06 4969.29 | 2483.33    0.0    323     0 |    1.86   0.19  37.52   0.02
3  1.64e+06 4969.29 | 2687.55    0.0    354     0 |    1.93   0.19  38.44   0.02
3  1.65e+06 4969.29 | 4662.24    0.0    574     0 |    1.90   0.18  38.91   0.02
3  1.66e+06 4969.29 | 2818.31    0.0    363     0 |    1.98   0.19  38.51   0.02
3  1.68e+06 4969.29 | 1649.55    0.0    219     0 |    1.84   0.20  37.76   0.02
3  1.69e+06 4969.29 | 4783.08    0.0    580     0 |    1.92   0.19  39.04   0.02
3  1.70e+06 4969.29 |  406.79    0.0     65     0 |    1.89   0.19  37.27   0.02
3  1.71e+06 4969.29 | 2765.69    0.0    338     0 |    1.92   0.20  38.05   0.02
3  1.72e+06 4969.29 | 1396.78    0.0    179     0 |    1.92   0.19  37.59   0.02
3  1.73e+06 4969.29 | 1609.04    0.0    209     0 |    1.92   0.20  37.77   0.02
3  1.74e+06 4969.29 | 4467.71    0.0    560     0 |    1.96   0.19  38.12   0.02
3  1.75e+06 5073.69 |
3  1.75e+06 5073.69 | 5073.69 3289.1    624   396 |    1.97   0.19  38.57   0.02
3  1.76e+06 5073.69 | 2309.20    0.0    281     0 |    1.96   0.19  39.72   0.02
3  1.77e+06 7728.06 |
3  1.77e+06 7728.06 | 7728.06  851.9    914    95 |    1.93   0.19  38.74   0.02
3  1.78e+06 7728.06 | 1083.30    0.0    144     0 |    2.01   0.19  39.00   0.02
3  1.79e+06 7728.06 | 4592.06    0.0    551     0 |    1.94   0.19  39.21   0.02
3  1.80e+06 7728.06 | 1914.95    0.0    254     0 |    2.03   0.19  39.01   0.02
3  1.81e+06 7728.06 |  708.27    0.0     99     0 |    1.99   0.19  38.98   0.02
3  1.82e+06 7728.06 | 7678.90 1510.8    896   179 |    2.00   0.19  38.46   0.02
3  1.83e+06 7728.06 | 4612.89    0.0    554     0 |    2.05   0.20  40.58   0.02
3  1.84e+06 7728.06 | 5765.75 3283.5    670   370 |    2.01   0.19  39.72   0.02
3  1.85e+06 7728.06 | 5138.14    0.0    609     0 |    2.03   0.20  41.17   0.02
3  1.86e+06 7728.06 | 2317.73    0.0    282     0 |    2.04   0.19  39.38   0.02
3  1.87e+06 7728.06 | 4502.98    0.0    518     0 |    1.92   0.20  39.25   0.02
3  1.88e+06 7728.06 |  434.40    0.0     66     0 |    2.05   0.19  39.97   0.02
3  1.89e+06 7728.06 | 2278.33    0.0    304     0 |    1.99   0.19  40.53   0.02
3  1.90e+06 7728.06 | 1075.64    0.0    139     0 |    1.99   0.21  40.48   0.02
3  1.91e+06 7728.06 | 5303.26 1961.5    643   238 |    2.03   0.19  40.77   0.02
3  1.92e+06 7728.06 | 7610.01 1916.9    876   215 |    1.96   0.19  39.49   0.02
3  1.93e+06 7728.06 | 7504.85 1400.1    897   159 |    1.94   0.19  41.41   0.02
3  1.94e+06 7728.06 | 7434.55 1282.9    860   141 |    2.00   0.19  40.33   0.02
3  1.95e+06 7728.06 | 1779.93    0.0    215     0 |    2.07   0.19  40.77   0.02
3  1.96e+06 7728.06 |  525.02    0.0     76     0 |    2.04   0.20  41.53   0.02
3  1.97e+06 7728.06 |  459.73    0.0     71     0 |    2.06   0.19  39.72   0.02
3  1.98e+06 7728.06 | 4457.85 4074.2    524   466 |    2.05   0.19  42.10   0.02
3  1.99e+06 7728.06 | 6060.01 2013.8    717   224 |    2.02   0.19  41.74   0.02
3  2.00e+06 7728.06 | 3763.48    0.0    434     0 |    1.99   0.19  42.00   0.02
3  2.01e+06 7728.06 | 2920.24 3370.0    346   379 |    2.00   0.20  40.89   0.02
3  2.03e+06 7728.06 | 5806.59    0.0    662     0 |    2.05   0.19  42.19   0.02
3  2.03e+06 7728.06 | 6983.07 3317.2    789   365 |    2.07   0.20  41.56   0.02
3  2.05e+06 7728.06 | 2810.41    0.0    331     0 |    2.06   0.20  41.36   0.02
3  2.06e+06 7728.06 | 7325.62 1785.0    814   193 |    2.04   0.19  42.16   0.02
3  2.07e+06 7728.06 | 5472.22    0.0    624     0 |    2.08   0.19  41.91   0.02
3  2.08e+06 7728.06 | 5224.22 2573.9    596   285 |    2.06   0.20  42.81   0.02
3  2.08e+06 7728.06 | 3663.78    0.0    424     0 |    2.04   0.20  41.13   0.02
3  2.09e+06 7788.26 |
3  2.09e+06 7788.26 | 7788.26 1759.1    876   189 |    2.00   0.19  41.34   0.02
3  2.10e+06 7830.41 |
3  2.10e+06 7830.41 | 7830.41 1640.3    890   190 |    2.11   0.19  43.17   0.02
3  2.11e+06 7830.41 | 2744.79    0.0    334     0 |    2.07   0.20  41.95   0.02
3  2.12e+06 7830.41 |  338.01    0.0     52     0 |    2.08   0.19  43.14   0.02
3  2.13e+06 7830.41 |  378.62    0.0     57     0 |    1.95   0.19  41.59   0.02
3  2.14e+06 7830.41 |  193.63    0.0     34     0 |    1.88   0.19  42.16   0.02
3  2.15e+06 8726.12 |
3  2.15e+06 8726.12 | 8726.12  592.7    966    58 |    2.12   0.19  43.00   0.02
3  2.16e+06 8726.12 |  180.76    0.0     31     0 |    2.09   0.18  41.18   0.02
3  2.17e+06 8726.12 | 5604.62 3839.7    616   407 |    2.10   0.19  43.04   0.02
3  2.18e+06 8726.12 | 8339.71    0.0    932     0 |    2.09   0.19  42.43   0.02
3  2.20e+06 8726.12 | 2646.60    0.0    300     0 |    2.11   0.18  44.49   0.02
3  2.20e+06 8726.12 | 5454.00 2853.0    618   313 |    1.92   0.19  43.55   0.02
3  2.21e+06 8726.12 | 4760.50    0.0    523     0 |    1.93   0.19  42.93   0.02
3  2.22e+06 9094.07 |
3  2.22e+06 9094.07 | 9094.07   80.9   1000     0 |    2.10   0.18  43.27   0.02
3  2.23e+06 9094.07 | 6217.18    0.0    674     0 |    2.03   0.19  44.04   0.02
3  2.24e+06 9094.07 | 8890.46    0.0   1000     0 |    2.19   0.18  44.23   0.02
3  2.25e+06 9400.97 |
3  2.25e+06 9400.97 | 9400.97   79.1   1000     0 |    2.12   0.18  44.17   0.02
3  2.26e+06 9400.97 | 9185.07    0.0   1000     0 |    2.20   0.18  43.76   0.02
3  2.27e+06 9400.97 | 9208.88    0.0   1000     0 |    2.01   0.18  44.20   0.02
3  2.28e+06 9400.97 | 8215.99 1686.5    900   173 |    2.21   0.18  43.55   0.02
3  2.29e+06 9400.97 | 7827.44    0.0    840     0 |    2.21   0.19  43.74   0.02
3  2.30e+06 9400.97 |  567.31    0.0     78     0 |    2.18   0.18  44.26   0.02
3  2.31e+06 9400.97 | 4230.70    0.0    468     0 |    2.07   0.18  44.48   0.02
3  2.32e+06 9400.97 |  336.20    0.0     52     0 |    2.03   0.19  44.77   0.02
3  2.33e+06 9400.97 | 9327.56    0.0   1000     0 |    1.94   0.19  44.30   0.02
3  2.34e+06 9400.97 | 2688.67 3918.1    296   409 |    2.09   0.18  45.14   0.02
3  2.35e+06 9400.97 |  270.88    0.0     43     0 |    2.09   0.18  44.38   0.02
3  2.36e+06 9400.97 | 4219.46    0.0    457     0 |    2.10   0.18  46.13   0.02
3  2.37e+06 9400.97 |  934.99    0.0    118     0 |    2.09   0.19  45.69   0.02
3  2.38e+06 9400.97 | 9338.99    0.0   1000     0 |    2.06   0.18  45.48   0.02
3  2.39e+06 9400.97 | 5507.83    0.0    584     0 |    1.92   0.18  46.07   0.02
3  2.40e+06 9400.97 | 3810.83    0.0    412     0 |    1.90   0.19  44.47   0.02
3  2.41e+06 9400.97 | 4665.77    0.0    506     0 |    1.99   0.18  45.91   0.02
3  2.42e+06 9400.97 | 1511.40    0.0    174     0 |    1.95   0.18  46.20   0.02
3  2.42e+06 9400.97 | 2301.35    0.0    255     0 |    1.97   0.18  46.06   0.02
3  2.43e+06 9400.97 | 1289.04    0.0    157     0 |    2.14   0.18  46.75   0.02
3  2.44e+06 9400.97 | 3766.25    0.0    425     0 |    2.09   0.18  46.02   0.02
3  2.45e+06 9400.97 | 1576.04    0.0    188     0 |    2.18   0.18  46.27   0.02
3  2.46e+06 9400.97 |  338.50    0.0     51     0 |    2.21   0.19  46.15   0.02
3  2.47e+06 9400.97 | 1363.64    0.0    162     0 |    2.02   0.19  46.61   0.02
3  2.48e+06 9400.97 | 2311.31    0.0    260     0 |    2.12   0.20  47.16   0.02
3  2.49e+06 9400.97 | 1103.78    0.0    138     0 |    2.30   0.19  46.52   0.02
3  2.50e+06 9400.97 | 1327.52    0.0    156     0 |    2.16   0.19  45.45   0.02
3  2.51e+06 9400.97 | 6124.39 2098.5    640   210 |    2.23   0.19  47.28   0.02
3  2.52e+06 9400.97 | 3947.05    0.0    427     0 |    2.23   0.20  47.51   0.02
3  2.53e+06 9400.97 |  365.90    0.0     55     0 |    2.26   0.18  47.74   0.02
3  2.54e+06 9400.97 | 3208.60    0.0    341     0 |    1.93   0.19  48.40   0.02
3  2.55e+06 9400.97 | 7360.38 2959.7    748   289 |    2.12   0.18  47.52   0.02
3  2.56e+06 9400.97 | 6598.35 2526.5    693   255 |    2.25   0.20  47.14   0.02
3  2.57e+06 9400.97 | 4869.49 4685.1    516   484 |    2.32   0.20  47.71   0.02
3  2.58e+06 9400.97 | 8345.87 2688.1    847   265 |    2.25   0.20  48.28   0.02
3  2.59e+06 9400.97 | 2920.38    0.0    314     0 |    2.28   0.19  47.91   0.02
3  2.60e+06 9400.97 | 5261.52    0.0    550     0 |    2.27   0.19  48.32   0.02
3  2.60e+06 9400.97 | 2043.06    0.0    223     0 |    2.21   0.20  48.21   0.02
3  2.61e+06 9400.97 | 4010.91    0.0    433     0 |    2.10   0.20  48.61   0.02
3  2.62e+06 9400.97 | 3848.38    0.0    408     0 |    2.30   0.20  48.81   0.02
3  2.63e+06 9400.97 |  389.13    0.0     58     0 |    2.29   0.19  47.63   0.02
3  2.64e+06 9400.97 |  365.81    0.0     54     0 |    2.16   0.20  48.71   0.02
3  2.65e+06 9400.97 | 3056.22    0.0    328     0 |    2.31   0.20  48.48   0.02
3  2.66e+06 9400.97 |  889.49    0.0    111     0 |    2.08   0.19  48.70   0.02
3  2.67e+06 9400.97 |  165.39    0.0     29     0 |    2.20   0.19  48.89   0.02
3  2.68e+06 9400.97 | 3100.69    0.0    360     0 |    2.08   0.21  49.12   0.02
3  2.68e+06 9400.97 |  498.45    0.0     69     0 |    2.28   0.20  49.01   0.02
3  2.69e+06 9400.97 | 1110.05    0.0    131     0 |    1.95   0.20  48.85   0.02
3  2.70e+06 9400.97 | 3564.61    0.0    367     0 |    2.09   0.20  49.62   0.02
3  2.71e+06 9400.97 | 3891.63    0.0    418     0 |    2.23   0.20  49.66   0.02
3  2.72e+06 9400.97 | 3446.70    0.0    368     0 |    2.19   0.20  50.11   0.02
3  2.73e+06 9400.97 | 2262.99    0.0    240     0 |    2.12   0.21  50.37   0.02
3  2.74e+06 9400.97 |  316.87    0.0     48     0 |    2.09   0.21  49.94   0.02
3  2.75e+06 9400.97 | 8746.19    0.0    877     0 |    2.02   0.21  49.60   0.02
3  2.76e+06 9400.97 | 2905.47    0.0    312     0 |    2.29   0.22  49.86   0.02
3  2.76e+06 9400.97 | 3270.80    0.0    336     0 |    2.07   0.20  50.74   0.02
3  2.77e+06 9400.97 | 2539.63    0.0    265     0 |    2.19   0.22  48.64   0.02
3  2.78e+06 9400.97 | 5086.21    0.0    524     0 |    2.23   0.21  50.41   0.02
3  2.79e+06 9400.97 | 6139.51    0.0    612     0 |    2.38   0.22  51.37   0.02
3  2.80e+06 9400.97 | 1076.67    0.0    128     0 |    2.20   0.22  49.96   0.02
3  2.81e+06 9400.97 | 7106.03    0.0    682     0 |    2.12   0.20  50.93   0.02
3  2.82e+06 9400.97 |  828.86    0.0    103     0 |    2.27   0.21  50.35   0.02
3  2.83e+06 9400.97 | 2867.47    0.0    300     0 |    2.06   0.21  50.02   0.02
3  2.84e+06 9400.97 |  439.11    0.0     62     0 |    2.29   0.21  51.19   0.02
3  2.85e+06 9400.97 | 4167.51    0.0    421     0 |    2.19   0.20  50.30   0.02
3  2.85e+06 9400.97 | 2720.88    0.0    279     0 |    2.19   0.21  51.17   0.02
3  2.86e+06 9400.97 | 3175.01    0.0    323     0 |    2.24   0.22  50.80   0.02
3  2.87e+06 9400.97 | 1955.77    0.0    211     0 |    2.06   0.22  50.54   0.02
3  2.88e+06 9400.97 | 3316.37    0.0    341     0 |    2.13   0.21  50.84   0.02
3  2.89e+06 9400.97 |  722.73    0.0    108     0 |    2.22   0.26  52.52   0.02
3  2.90e+06 9400.97 |  189.10    0.0     37     0 |    1.33   0.23  51.74   0.02
3  2.91e+06 9400.97 | 2395.07    0.0    252     0 |    1.88   0.22  51.33   0.02
3  2.92e+06 9400.97 | 2206.58    0.0    238     0 |    2.04   0.21  52.45   0.02
3  2.93e+06 9400.97 |  260.01    0.0     42     0 |    2.14   0.21  50.83   0.02
3  2.93e+06 9400.97 | 9167.51    0.0    892     0 |    2.11   0.22  52.46   0.02
3  2.94e+06 9400.97 | 2142.60    0.0    235     0 |    2.14   0.21  51.50   0.02
3  2.95e+06 9400.97 |  663.20    0.0     85     0 |    2.32   0.22  51.99   0.02
3  2.96e+06 9400.97 | 6153.73    0.0    614     0 |    2.20   0.21  52.28   0.02
3  2.97e+06 9400.97 | 1746.93    0.0    184     0 |    2.33   0.22  52.50   0.02
3  2.98e+06 9400.97 | 4008.55    0.0    411     0 |    2.14   0.22  51.22   0.02
3  2.99e+06 9400.97 |  323.49    0.0     49     0 |    2.23   0.23  52.69   0.02
3  3.00e+06 9400.97 | 6496.10    0.0    629     0 |    2.28   0.24  52.76   0.02
3  3.01e+06 9400.97 |  839.82    0.0    103     0 |    2.35   0.23  52.15   0.02
3  3.02e+06 9400.97 | 6746.13 3928.6    648   359 |    2.22   0.23  52.27   0.02
3  3.02e+06 9400.97 | 2614.53    0.0    273     0 |    2.40   0.21  51.97   0.02
3  3.03e+06 9400.97 | 6482.44 2960.4    632   274 |    2.29   0.22  52.42   0.02
3  3.04e+06 9400.97 | 2710.71    0.0    272     0 |    2.38   0.23  53.30   0.02
3  3.05e+06 9400.97 | 2630.47    0.0    279     0 |    2.26   0.22  52.33   0.02
3  3.06e+06 9400.97 | 6490.97    0.0    627     0 |    2.41   0.22  53.16   0.02
3  3.07e+06 9400.97 | 5756.57 3020.6    558   278 |    2.20   0.21  53.43   0.02
3  3.08e+06 9400.97 | 2955.34    0.0    301     0 |    2.42   0.22  53.79   0.02
3  3.09e+06 9400.97 |  545.33    0.0     74     0 |    2.27   0.23  52.33   0.02
3  3.10e+06 9400.97 | 2198.62    0.0    229     0 |    2.35   0.22  53.38   0.02
3  3.11e+06 9400.97 | 3655.70    0.0    362     0 |    2.25   0.22  53.73   0.02
3  3.12e+06 9400.97 | 7572.88    0.0    688     0 |    2.40   0.23  53.36   0.02
3  3.13e+06 9400.97 |  686.92    0.0     88     0 |    2.31   0.25  53.30   0.02
3  3.14e+06 9400.97 | 4784.15    0.0    470     0 |    2.34   0.22  53.50   0.02
3  3.15e+06 9400.97 | 6257.92    0.0    597     0 |    2.40   0.23  54.19   0.02
3  3.16e+06 9400.97 |  381.11    0.0     56     0 |    2.55   0.22  54.06   0.02
3  3.17e+06 9400.97 | 4226.98    0.0    408     0 |    2.36   0.23  53.91   0.02
3  3.18e+06 9400.97 | 4583.93    0.0    446     0 |    2.33   0.23  54.37   0.02
3  3.19e+06 9400.97 | 6077.15 3476.0    576   311 |    2.27   0.24  54.81   0.02
3  3.20e+06 9400.97 | 5583.01 3175.2    534   278 |    2.34   0.22  54.06   0.02
3  3.21e+06 9400.97 | 6684.09 2633.0    644   234 |    2.33   0.25  54.21   0.02
3  3.22e+06 9400.97 | 2594.13    0.0    277     0 |    2.45   0.24  54.45   0.02
3  3.23e+06 9400.97 | 3951.93    0.0    381     0 |    2.39   0.23  54.81   0.02
3  3.24e+06 9400.97 | 6620.14    0.0    618     0 |    2.37   0.23  54.77   0.02
3  3.24e+06 9400.97 | 3264.37    0.0    323     0 |    2.38   0.25  55.88   0.02
3  3.25e+06 9400.97 | 3717.48    0.0    369     0 |    2.28   0.23  54.36   0.02
3  3.26e+06 9400.97 | 6228.22 3159.0    590   283 |    2.39   0.24  55.27   0.02
3  3.27e+06 9400.97 | 3191.35    0.0    313     0 |    2.42   0.25  53.25   0.02
3  3.28e+06 9400.97 | 6677.04    0.0    652     0 |    2.38   0.24  56.30   0.02
3  3.29e+06 9400.97 | 1202.97    0.0    135     0 |    2.34   0.23  55.98   0.02
3  3.30e+06 9400.97 | 3834.14    0.0    378     0 |    2.26   0.23  55.60   0.02
3  3.31e+06 9400.97 | 7547.63    0.0    715     0 |    2.28   0.25  56.10   0.02
3  3.32e+06 9400.97 | 3160.17    0.0    305     0 |    2.39   0.25  55.70   0.02
3  3.33e+06 9400.97 | 3366.77    0.0    330     0 |    2.20   0.26  56.55   0.02
3  3.34e+06 9400.97 | 2180.08    0.0    226     0 |    2.51   0.25  55.42   0.02
3  3.35e+06 9400.97 | 4012.17 3974.3    392   356 |    2.43   0.26  55.38   0.02
3  3.35e+06 9400.97 | 2136.97    0.0    221     0 |    2.27   0.25  56.35   0.02
3  3.36e+06 9400.97 | 2565.83    0.0    254     0 |    2.27   0.24  56.11   0.02
3  3.38e+06 9400.97 | 1909.27    0.0    205     0 |    2.47   0.25  56.66   0.02
3  3.38e+06 9400.97 | 4336.54    0.0    418     0 |    2.46   0.26  56.28   0.02
3  3.39e+06 9400.97 | 5321.48 3593.4    494   307 |    2.40   0.25  56.29   0.02
3  3.40e+06 9400.97 | 2623.19    0.0    280     0 |    2.33   0.24  56.03   0.02
3  3.41e+06 9400.97 | 4583.58    0.0    446     0 |    2.37   0.26  56.75   0.02
3  3.42e+06 9400.97 | 2202.31    0.0    234     0 |    2.39   0.25  56.72   0.02
3  3.43e+06 9400.97 | 4711.35    0.0    450     0 |    2.34   0.25  57.02   0.02
3  3.44e+06 9400.97 |  222.00    0.0     36     0 |    2.40   0.26  57.51   0.02
3  3.45e+06 9400.97 |  583.82    0.0     95     0 |    2.51   0.32  60.83   0.03
3  3.46e+06 9400.97 | 1659.08    0.0    180     0 |    1.55   0.24  57.01   0.02
3  3.47e+06 9400.97 | 4795.89    0.0    475     0 |    2.46   0.25  56.82   0.02
3  3.48e+06 9400.97 | 3277.17    0.0    314     0 |    2.45   0.26  56.12   0.02
3  3.49e+06 9400.97 | 6135.82 2896.6    584   257 |    2.48   0.26  57.23   0.02
3  3.50e+06 9400.97 | 2316.13    0.0    239     0 |    2.48   0.24  56.64   0.02
3  3.51e+06 9400.97 | 2459.48    0.0    243     0 |    2.46   0.26  57.63   0.02
3  3.52e+06 9400.97 |  545.49    0.0     72     0 |    2.42   0.27  57.16   0.02
3  3.53e+06 9400.97 | 3056.77    0.0    315     0 |    2.49   0.26  56.49   0.02
3  3.54e+06 9400.97 | 3525.36    0.0    334     0 |    2.51   0.26  56.60   0.02
3  3.55e+06 9400.97 |  448.69    0.0     62     0 |    2.40   0.25  57.04   0.02
3  3.56e+06 9400.97 | 4425.29    0.0    417     0 |    2.46   0.25  58.06   0.02
3  3.57e+06 9400.97 | 7055.81    0.0    647     0 |    2.42   0.25  56.89   0.02
3  3.58e+06 9400.97 | 3547.77    0.0    343     0 |    2.40   0.24  57.90   0.02
3  3.59e+06 9400.97 | 1214.78    0.0    140     0 |    2.51   0.27  57.81   0.02
3  3.60e+06 9400.97 | 1943.99    0.0    203     0 |    2.43   0.26  57.63   0.02
3  3.60e+06 9400.97 | 2539.23    0.0    255     0 |    2.46   0.24  57.93   0.02
3  3.61e+06 9400.97 | 5755.68    0.0    533     0 |    2.44   0.27  57.88   0.02
3  3.62e+06 9400.97 |  521.56    0.0     69     0 |    2.38   0.25  57.91   0.02
3  3.63e+06 9400.97 | 2260.82    0.0    224     0 |    2.49   0.26  57.59   0.02
3  3.64e+06 9400.97 | 8032.42 3036.8    733   268 |    2.35   0.26  57.60   0.02
3  3.65e+06 9400.97 | 5434.70    0.0    494     0 |    2.45   0.27  58.28   0.02
3  3.66e+06 9400.97 | 3497.88    0.0    334     0 |    2.29   0.26  57.95   0.02
3  3.67e+06 9400.97 | 5385.06    0.0    530     0 |    2.36   0.27  58.82   0.02
3  3.68e+06 9400.97 | 7716.17    0.0    712     0 |    2.38   0.24  58.36   0.02
3  3.69e+06 9400.97 | 7525.99    0.0    678     0 |    2.33   0.27  57.81   0.02
3  3.70e+06 9400.97 |  970.74    0.0    115     0 |    2.51   0.26  58.44   0.02
3  3.71e+06 9400.97 | 3055.52    0.0    303     0 |    2.47   0.27  59.08   0.02
3  3.72e+06 9400.97 | 6156.14    0.0    593     0 |    2.51   0.27  58.46   0.02
3  3.73e+06 9400.97 |  757.89    0.0     93     0 |    2.50   0.25  58.00   0.02
3  3.74e+06 9400.97 | 2969.20    0.0    291     0 |    2.45   0.26  59.13   0.02
3  3.75e+06 9400.97 | 6579.89 3733.9    612   332 |    2.49   0.26  59.74   0.02
3  3.76e+06 9400.97 | 4382.62 4036.6    412   359 |    2.47   0.27  58.02   0.02
3  3.77e+06 9400.97 | 6604.70    0.0    627     0 |    2.61   0.25  58.69   0.02
3  3.78e+06 9400.97 | 4470.17 3493.4    428   324 |    2.51   0.25  58.66   0.02
3  3.79e+06 9400.97 |  873.06    0.0    108     0 |    2.45   0.26  58.98   0.02
3  3.80e+06 9400.97 | 3919.33    0.0    369     0 |    2.54   0.28  58.55   0.02
3  3.81e+06 9400.97 | 8246.80 4263.3    739   365 |    2.47   0.28  59.17   0.02
3  3.82e+06 9400.97 | 5343.21    0.0    497     0 |    2.50   0.27  59.11   0.02
3  3.83e+06 9400.97 | 3814.69    0.0    364     0 |    2.57   0.27  59.73   0.02
3  3.84e+06 9400.97 |  206.73    0.0     41     0 |    2.46   0.28  59.52   0.02
3  3.85e+06 9400.97 | 1580.95    0.0    165     0 |    1.89   0.26  58.67   0.02
3  3.86e+06 9400.97 | 8301.75    0.0    744     0 |    2.23   0.27  58.73   0.02
3  3.87e+06 9400.97 | 5613.82 4133.1    514   350 |    2.58   0.26  59.09   0.02
3  3.88e+06 9400.97 | 2753.20    0.0    268     0 |    2.60   0.27  59.82   0.02
3  3.89e+06 9400.97 | 3457.35    0.0    327     0 |    2.55   0.28  59.33   0.02
3  3.90e+06 9400.97 |  174.46    0.0     30     0 |    2.48   0.27  59.32   0.02
3  3.91e+06 9400.97 |  428.61    0.0     60     0 |    2.57   0.28  58.55   0.02
3  3.92e+06 9400.97 | 6091.70    0.0    554     0 |    2.56   0.26  60.11   0.02
3  3.93e+06 9400.97 | 7670.87    0.0    710     0 |    2.66   0.28  60.68   0.02
3  3.94e+06 9400.97 | 2778.81    0.0    276     0 |    2.15   0.27  60.50   0.02
3  3.95e+06 9400.97 | 6145.78    0.0    570     0 |    2.58   0.27  59.34   0.02
3  3.96e+06 9400.97 | 6133.93    0.0    551     0 |    2.53   0.27  59.78   0.02
3  3.97e+06 9400.97 | 4161.56    0.0    388     0 |    2.58   0.27  60.53   0.02
3  3.98e+06 9400.97 | 3202.24    0.0    306     0 |    2.47   0.28  59.90   0.02
3  3.99e+06 9400.97 | 5751.28 3699.0    526   318 |    2.62   0.28  60.64   0.02
3  4.00e+06 9400.97 | 9008.62    0.0    876     0 |    2.64   0.27  60.58   0.02
| UsedTime:   49475 | SavedDir: ./Humanoid-v3_ReliableSAC_3
| Learner: Save in ./Humanoid-v3_ReliableSAC_3
        """
    elif env_name == 'Humanoid-v3.8140':
        from elegantrl.envs.CustomGymEnv import HumanoidEnv
        env_func = HumanoidEnv
        env_args = {
            'env_num': 1,
            'env_name': 'Humanoid-v3',
            'max_step': 1000,
            'state_dim': 376,
            'action_dim': 17,
            'if_discrete': False,
            'target_return': 3000.,
        }
        args = Arguments(agent, env_func=env_func, env_args=env_args)

        args.eval_times = 2 ** 2
        args.reward_scale = 2 ** -2
        args.max_memo = int(2 ** 21 * 1.5)
        args.learning_rate = 2 ** -14
        args.lambda_a_log_std = 2 ** -6

        args.target_step = args.max_step
        args.worker_num = 4

        args.net_dim = 2 ** 9
        args.batch_size = args.net_dim // 2

        args.gamma = 0.985
        args.num_layer = 3
        args.repeat_times = 2 ** 0
        args.if_act_target = False
        import numpy as np
        args.target_entropy = np.log(env_args['action_dim'])  # todo

        args.if_allow_break = False
        args.break_step = int(4e6)

        """
        | Arguments Remove cwd: ./Humanoid-v3_ReliableSAC_2
################################################################################
ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
2  8.10e+03   90.78 |
2  8.10e+03   90.78 |   90.78    9.5     18     2 |    1.20   0.69   2.78   0.06
2  1.14e+05  227.95 |
2  1.14e+05  227.95 |  227.95    8.4     44     2 |    1.27   0.79  26.12   0.04
2  1.76e+05  240.13 |
2  1.76e+05  240.13 |  240.13   10.1     45     2 |    1.32   0.77  33.01   0.02
2  2.26e+05  264.62 |
2  2.26e+05  264.62 |  264.62   20.5     49     3 |    1.34   0.63  32.48   0.02
2  2.67e+05  264.62 |  219.66    0.0     42     0 |    1.34   0.53  28.57   0.02
2  3.04e+05  341.61 |
2  3.04e+05  341.61 |  341.61   25.4     64     4 |    1.33   0.39  25.56   0.02
2  3.37e+05  341.61 |  260.65    0.0     55     0 |    1.32   0.32  25.71   0.01
2  3.66e+05  413.34 |
2  3.66e+05  413.34 |  413.34   40.6     78     8 |    1.30   0.30  26.26   0.01
2  3.91e+05  413.34 |  278.77    0.0     53     0 |    1.34   0.27  26.03   0.01
2  4.16e+05  413.34 |  274.65    0.0     54     0 |    1.35   0.25  26.42   0.01
2  4.41e+05  557.19 |
2  4.41e+05  557.19 |  557.19  107.9    108    16 |    1.35   0.24  25.52   0.01
2  4.66e+05  557.19 |  392.51    0.0     73     0 |    1.32   0.22  25.35   0.01
2  4.87e+05  557.19 |  385.15    0.0     70     0 |    1.36   0.21  24.85   0.01
2  5.08e+05  557.19 |  426.10    0.0     79     0 |    1.32   0.21  25.51   0.01
2  5.29e+05  557.19 |  399.47    0.0     71     0 |    1.34   0.20  25.57   0.01
2  5.50e+05  557.19 |  385.00    0.0     71     0 |    1.36   0.19  25.35   0.01
2  5.71e+05  557.19 |  435.28    0.0     76     0 |    1.36   0.19  25.96   0.01
2  5.87e+05  557.19 |  466.93    0.0     84     0 |    1.34   0.19  25.93   0.01
2  6.04e+05  557.19 |  470.94    0.0     84     0 |    1.39   0.18  26.30   0.01
2  6.21e+05  557.19 |  402.99    0.0     72     0 |    1.37   0.19  25.32   0.01
2  6.37e+05  557.19 |  385.18    0.0     67     0 |    1.39   0.18  26.13   0.01
2  6.54e+05  557.19 |  524.94    0.0     95     0 |    1.41   0.18  26.40   0.01
2  6.70e+05  557.19 |  540.23    0.0     95     0 |    1.38   0.17  27.68   0.01
2  6.87e+05  557.19 |  422.21    0.0     75     0 |    1.40   0.18  28.46   0.01
2  7.04e+05  557.19 |  291.22    0.0     55     0 |    1.40   0.17  27.04   0.01
2  7.21e+05  557.19 |  373.98    0.0     66     0 |    1.38   0.17  27.55   0.01
2  7.38e+05  557.19 |  531.67   83.8     95    15 |    1.36   0.17  27.87   0.01
2  7.54e+05  557.19 |  411.16    0.0     73     0 |    1.38   0.17  27.85   0.01
2  7.72e+05  663.21 |
2  7.72e+05  663.21 |  663.21  186.4    119    30 |    1.40   0.17  28.84   0.01
2  7.84e+05  663.21 |  530.18    0.0     94     0 |    1.39   0.17  27.86   0.01
2  7.97e+05  663.21 |  537.98    0.0     96     0 |    1.41   0.17  29.90   0.01
2  8.10e+05  765.51 |
2  8.10e+05  765.51 |  765.51   74.8    133    16 |    1.38   0.18  28.83   0.02
2  8.23e+05  765.51 |  746.94    0.0    132     0 |    1.38   0.16  28.68   0.02
2  8.36e+05  765.51 |  644.94    0.0    116     0 |    1.38   0.17  29.29   0.02
2  8.49e+05  765.51 |  754.08    0.0    126     0 |    1.39   0.17  28.72   0.02
2  8.62e+05  765.51 |  503.48    0.0     94     0 |    1.41   0.18  29.79   0.02
2  8.75e+05  765.51 |  537.22    0.0     99     0 |    1.44   0.18  30.48   0.02
2  8.87e+05  765.51 |  659.88    0.0    111     0 |    1.43   0.18  29.34   0.02
2  9.00e+05  765.51 |  761.09  177.9    129    28 |    1.46   0.18  30.42   0.02
2  9.13e+05  765.51 |  613.23    0.0    106     0 |    1.46   0.17  31.09   0.02
2  9.26e+05 1038.18 |
2  9.26e+05 1038.18 | 1038.18  227.4    177    34 |    1.45   0.17  29.79   0.02
2  9.39e+05 1038.18 |  915.45  311.8    154    51 |    1.44   0.17  30.75   0.02
2  9.52e+05 1038.18 |  695.13    0.0    119     0 |    1.49   0.18  30.36   0.02
2  9.65e+05 1038.18 |  646.47    0.0    118     0 |    1.50   0.18  30.80   0.02
2  9.78e+05 1038.18 |  974.30  299.1    163    49 |    1.48   0.18  30.86   0.02
2  9.91e+05 1038.18 |  604.55    0.0    103     0 |    1.49   0.18  31.58   0.02
2  1.00e+06 1038.18 |  922.67    0.0    152     0 |    1.48   0.18  30.21   0.02
2  1.02e+06 1038.18 |  673.91    0.0    106     0 |    1.50   0.18  31.29   0.02
2  1.03e+06 1038.18 |  982.81  297.8    160    49 |    1.52   0.18  31.76   0.02
2  1.04e+06 1038.18 |  855.41    0.0    136     0 |    1.53   0.18  32.46   0.02
2  1.06e+06 1824.70 |
2  1.06e+06 1824.70 | 1824.70  704.9    285   105 |    1.53   0.19  32.25   0.02
2  1.07e+06 1824.70 | 1319.13    0.0    207     0 |    1.53   0.18  32.58   0.02
2  1.08e+06 1824.70 | 1672.06    0.0    267     0 |    1.52   0.19  32.78   0.02
2  1.10e+06 1824.70 | 1325.30    0.0    207     0 |    1.58   0.19  33.83   0.02
2  1.11e+06 1824.70 | 1457.98  470.0    224    71 |    1.59   0.19  32.09   0.02
2  1.13e+06 1881.31 |
2  1.13e+06 1881.31 | 1881.31  714.1    289   100 |    1.58   0.19  34.07   0.02
2  1.14e+06 1881.31 | 1377.06    0.0    215     0 |    1.59   0.19  32.81   0.02
2  1.15e+06 1881.31 | 1335.53    0.0    205     0 |    1.63   0.19  32.02   0.02
2  1.17e+06 2130.46 |
2  1.17e+06 2130.46 | 2130.46  873.7    320   127 |    1.57   0.20  32.94   0.02
2  1.18e+06 2130.46 | 2002.78    0.0    292     0 |    1.59   0.20  34.26   0.02
2  1.19e+06 2130.46 | 1118.43    0.0    172     0 |    1.61   0.19  33.04   0.02
2  1.20e+06 2130.46 | 1210.14    0.0    187     0 |    1.62   0.19  34.59   0.02
2  1.21e+06 2130.46 |  889.89    0.0    139     0 |    1.66   0.19  34.45   0.02
2  1.22e+06 2211.02 |
2  1.22e+06 2211.02 | 2211.02 1027.2    336   144 |    1.62   0.19  33.81   0.02
2  1.23e+06 2211.02 |  955.95    0.0    149     0 |    1.67   0.19  34.10   0.02
2  1.24e+06 2211.02 | 1470.51    0.0    240     0 |    1.63   0.19  34.39   0.02
2  1.24e+06 2211.02 |  783.39    0.0    130     0 |    1.64   0.20  34.20   0.02
2  1.25e+06 2211.02 |  952.79    0.0    152     0 |    1.64   0.20  34.21   0.02
2  1.26e+06 2211.02 | 1436.46    0.0    230     0 |    1.63   0.20  34.65   0.02
2  1.27e+06 2211.02 | 1801.60    0.0    267     0 |    1.68   0.19  35.56   0.02
2  1.28e+06 4286.17 |
2  1.28e+06 4286.17 | 4286.17 2104.8    626   297 |    1.65   0.19  33.72   0.02
2  1.29e+06 4286.17 |  716.03    0.0    114     0 |    1.66   0.20  35.65   0.02
2  1.30e+06 4286.17 | 1972.81    0.0    300     0 |    1.66   0.19  35.24   0.02
2  1.31e+06 4286.17 | 1881.28    0.0    277     0 |    1.66   0.20  34.50   0.02
2  1.32e+06 4286.17 | 1420.99    0.0    213     0 |    1.67   0.19  35.51   0.02
2  1.33e+06 4286.17 | 2819.84    0.0    420     0 |    1.66   0.19  33.99   0.02
2  1.34e+06 4286.17 | 3262.76    0.0    466     0 |    1.66   0.19  35.15   0.02
2  1.35e+06 4286.17 | 4233.49 2461.5    600   332 |    1.71   0.19  34.08   0.02
2  1.36e+06 5612.00 |
2  1.36e+06 5612.00 | 5612.00 2030.8    778   275 |    1.69   0.19  35.28   0.02
2  1.37e+06 5612.00 | 1923.94    0.0    275     0 |    1.73   0.20  36.38   0.02
2  1.38e+06 5612.00 | 3570.13 2566.5    492   342 |    1.74   0.19  35.00   0.02
2  1.39e+06 5612.00 | 1569.53    0.0    219     0 |    1.67   0.20  35.52   0.02
2  1.40e+06 5612.00 | 2526.54    0.0    346     0 |    1.74   0.20  35.73   0.02
2  1.41e+06 7126.32 |
2  1.41e+06 7126.32 | 7126.32  249.9    980    35 |    1.70   0.20  36.92   0.02
2  1.42e+06 7126.32 |  463.55    0.0     74     0 |    1.74   0.19  36.85   0.02
2  1.43e+06 7126.32 | 5973.39    0.0    790     0 |    1.71   0.20  36.26   0.02
2  1.44e+06 7126.32 | 1466.78    0.0    207     0 |    1.74   0.20  37.44   0.02
2  1.45e+06 7126.32 | 5876.85 2582.9    800   346 |    1.78   0.19  37.04   0.02
2  1.46e+06 7331.79 |
2  1.46e+06 7331.79 | 7331.79  229.3    976    28 |    1.73   0.20  36.50   0.02
2  1.47e+06 7331.79 | 2664.43    0.0    366     0 |    1.76   0.19  37.18   0.02
2  1.48e+06 7331.79 | 1321.12    0.0    206     0 |    1.74   0.20  37.00   0.02
2  1.49e+06 7331.79 | 6368.94 2081.2    843   272 |    1.69   0.19  36.71   0.02
2  1.50e+06 7331.79 | 6667.05  876.8    887   118 |    1.79   0.20  37.61   0.02
2  1.51e+06 7331.79 | 5866.35    0.0    779     0 |    1.81   0.20  36.49   0.02
2  1.52e+06 7331.79 |  576.01    0.0     89     0 |    1.80   0.20  37.81   0.02
2  1.53e+06 7331.79 | 1688.60    0.0    233     0 |    1.77   0.20  37.29   0.02
2  1.54e+06 7331.79 | 5261.23 2435.0    700   319 |    1.77   0.20  37.64   0.02
2  1.55e+06 7331.79 | 5399.00    0.0    698     0 |    1.83   0.19  37.14   0.02
2  1.56e+06 7331.79 | 6157.42 1511.7    813   202 |    1.81   0.19  37.53   0.02
2  1.57e+06 7331.79 | 3162.86    0.0    410     0 |    1.82   0.19  37.39   0.02
2  1.58e+06 7331.79 | 3419.41    0.0    445     0 |    1.81   0.19  37.44   0.02
2  1.59e+06 7331.79 | 5282.29    0.0    709     0 |    1.83   0.20  37.65   0.02
2  1.59e+06 7331.79 |  445.82    0.0     73     0 |    1.75   0.19  37.69   0.02
2  1.60e+06 7331.79 | 5166.45 2919.7    662   364 |    1.75   0.20  37.73   0.02
2  1.61e+06 7331.79 | 1160.98    0.0    168     0 |    1.81   0.19  38.19   0.02
2  1.62e+06 7331.79 |  385.14    0.0     63     0 |    1.86   0.20  38.97   0.02
2  1.63e+06 7331.79 | 1296.00    0.0    177     0 |    1.76   0.20  38.75   0.02
2  1.64e+06 7331.79 | 3073.65    0.0    410     0 |    1.78   0.20  38.31   0.02
2  1.65e+06 7331.79 | 4470.72    0.0    591     0 |    1.81   0.20  39.00   0.02
2  1.66e+06 7331.79 |  333.14    0.0     56     0 |    1.79   0.20  39.33   0.02
2  1.67e+06 7331.79 | 5807.64 2430.3    741   302 |    1.85   0.19  37.65   0.02
2  1.68e+06 7331.79 | 4275.79 2694.9    561   335 |    1.89   0.19  39.04   0.02
2  1.69e+06 7331.79 | 1016.73    0.0    141     0 |    1.84   0.20  37.86   0.02
2  1.70e+06 7331.79 | 3605.95    0.0    471     0 |    1.77   0.20  38.81   0.02
2  1.71e+06 7331.79 |  736.54    0.0    107     0 |    1.83   0.20  39.10   0.02
2  1.72e+06 7331.79 |  996.17    0.0    141     0 |    1.74   0.20  38.28   0.02
2  1.73e+06 7331.79 | 4694.49 2852.3    600   358 |    1.85   0.20  38.42   0.02
2  1.74e+06 7331.79 | 2549.88    0.0    339     0 |    1.84   0.20  39.03   0.02
2  1.75e+06 7331.79 | 3115.12    0.0    396     0 |    1.76   0.20  40.11   0.02
2  1.76e+06 7331.79 | 1279.52    0.0    175     0 |    1.92   0.20  38.60   0.02
2  1.77e+06 7331.79 | 2957.39    0.0    386     0 |    1.84   0.20  40.33   0.02
2  1.78e+06 7331.79 | 1165.82    0.0    159     0 |    1.88   0.20  40.23   0.02
2  1.79e+06 7331.79 | 4275.53    0.0    520     0 |    1.86   0.20  39.45   0.02
2  1.80e+06 7331.79 | 5397.93    0.0    660     0 |    1.89   0.21  39.82   0.02
2  1.81e+06 7331.79 | 6072.07    0.0    738     0 |    1.91   0.21  37.96   0.02
2  1.82e+06 7331.79 | 2585.03    0.0    334     0 |    1.79   0.21  40.70   0.02
2  1.82e+06 7331.79 | 4416.40 2765.4    543   326 |    1.80   0.21  39.63   0.02
2  1.83e+06 7331.79 | 3399.75    0.0    432     0 |    1.88   0.21  40.33   0.02
2  1.84e+06 7331.79 | 6230.92    0.0    788     0 |    1.89   0.20  39.00   0.02
2  1.85e+06 7331.79 | 3259.26    0.0    410     0 |    1.87   0.21  40.84   0.02
2  1.86e+06 7331.79 | 1597.32    0.0    212     0 |    1.90   0.21  40.31   0.02
2  1.87e+06 7331.79 | 6815.74 1800.9    840   215 |    1.88   0.20  40.42   0.02
2  1.88e+06 7331.79 | 3414.72    0.0    436     0 |    1.93   0.20  40.98   0.02
2  1.89e+06 7331.79 |  571.97    0.0     87     0 |    1.84   0.21  41.38   0.02
2  1.90e+06 7331.79 | 3184.91    0.0    403     0 |    1.96   0.21  40.56   0.02
2  1.91e+06 8140.26 |
2  1.91e+06 8140.26 | 8140.26   84.2   1000     0 |    1.94   0.20  40.25   0.02
2  1.92e+06 8140.26 | 6630.22    0.0    803     0 |    1.91   0.21  42.25   0.02
2  1.93e+06 8140.26 | 5937.82    0.0    709     0 |    1.89   0.21  41.15   0.02
2  1.94e+06 8140.26 |  615.71    0.0     91     0 |    1.89   0.21  40.50   0.02
2  1.95e+06 8140.26 | 2642.54    0.0    355     0 |    2.01   0.22  40.92   0.02
2  1.96e+06 8140.26 | 1880.52    0.0    238     0 |    1.96   0.21  41.70   0.02
2  1.97e+06 8140.26 | 5411.17    0.0    644     0 |    1.91   0.21  40.88   0.02
2  1.98e+06 8140.26 | 3981.29    0.0    483     0 |    1.80   0.21  42.47   0.02
2  1.99e+06 8140.26 | 6557.22 3281.2    780   382 |    1.95   0.22  41.85   0.02
2  2.00e+06 8140.26 | 3041.03    0.0    371     0 |    1.91   0.21  41.13   0.02
        """
    elif env_name == 'Humanoid-v3.8605.best':
        from elegantrl.envs.CustomGymEnv import HumanoidEnv
        env_func = HumanoidEnv
        env_args = {
            'env_num': 1,
            'env_name': 'Humanoid-v3',
            'max_step': 1000,
            'state_dim': 376,
            'action_dim': 17,
            'if_discrete': False,
            'target_return': 3000.,
        }
        args = Arguments(agent, env_func=env_func, env_args=env_args)

        args.eval_times = 2 ** 2
        args.reward_scale = 2 ** -2
        args.max_memo = 2 ** 21
        args.learning_rate = 2 ** -14
        args.lambda_a_log_std = 2 ** -6

        args.target_step = args.max_step
        args.worker_num = 4

        args.net_dim = 2 ** 9
        args.batch_size = args.net_dim // 2
        args.gamma = 0.985  # todo
        args.num_layer = 3
        args.repeat_times = 2 ** 0
        args.if_act_target = False
        import numpy as np
        args.target_entropy = np.log(env_args['action_dim']) * 1.5  # todo

        args.if_allow_break = False
        args.break_step = int(2e6)
        """
| Arguments Remove cwd: ./Humanoid-v3_ReliableSAC_1
################################################################################
ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
1  8.10e+03  110.28 |
1  8.10e+03  110.28 |  110.28   21.3     22     4 |    1.18   0.68   1.68   0.06
1  1.43e+05  316.51 |
1  1.43e+05  316.51 |  316.51   56.3     69    10 |    1.30   0.78  29.82   0.03
1  2.17e+05  316.51 |  255.03    0.0     48     0 |    1.33   0.67  33.83   0.02
1  2.71e+05  337.53 |
1  2.71e+05  337.53 |  337.53   16.6     61     3 |    1.32   0.50  28.05   0.02
1  3.16e+05  337.53 |  275.44    0.0     51     0 |    1.32   0.36  26.29   0.01
1  3.57e+05  337.53 |  265.05    0.0     49     0 |    1.30   0.31  24.74   0.01
1  3.94e+05  414.59 |
1  3.94e+05  414.59 |  414.59   53.0     80     9 |    1.31   0.27  25.89   0.01
1  4.27e+05  414.59 |  278.19    0.0     51     0 |    1.32   0.25  24.99   0.01
1  4.60e+05  542.04 |
1  4.60e+05  542.04 |  542.04  136.9    104    28 |    1.34   0.23  26.24   0.01
1  4.89e+05  542.04 |  519.01    0.0     95     0 |    1.32   0.22  25.69   0.01
1  5.18e+05  542.04 |  309.69    0.0     68     0 |    1.35   0.21  25.27   0.01
1  5.43e+05  542.04 |  305.88    0.0     58     0 |    1.30   0.20  25.49   0.01
1  5.68e+05  542.04 |  266.79    0.0     56     0 |    1.37   0.19  25.52   0.01
1  5.93e+05  542.04 |  378.07    0.0     71     0 |    1.35   0.19  26.46   0.01
1  6.18e+05  542.04 |  524.26    0.0    110     0 |    1.38   0.19  25.63   0.01
1  6.43e+05  542.04 |  488.47    0.0     86     0 |    1.37   0.18  25.97   0.01
1  6.64e+05  542.04 |  437.22   71.1     89     7 |    1.35   0.18  27.04   0.01
1  6.84e+05  542.04 |  530.90    0.0     97     0 |    1.37   0.18  27.17   0.01
1  7.05e+05  542.04 |  349.59    0.0     64     0 |    1.38   0.17  27.22   0.01
1  7.26e+05  542.04 |  388.24    0.0     70     0 |    1.35   0.18  27.55   0.01
1  7.47e+05  542.04 |  514.88  112.4     90    19 |    1.38   0.17  27.94   0.01
1  7.68e+05  608.88 |
1  7.68e+05  608.88 |  608.88   72.7    109    12 |    1.37   0.17  27.06   0.01
1  7.89e+05  608.88 |  524.12    0.0     96     0 |    1.37   0.17  27.96   0.01
1  8.10e+05  648.58 |
1  8.10e+05  648.58 |  648.58   84.8    118    19 |    1.35   0.17  28.42   0.01
1  8.27e+05  648.58 |  584.14    0.0     98     0 |    1.37   0.17  27.81   0.01
1  8.44e+05  648.58 |  616.88    0.0    103     0 |    1.35   0.16  29.21   0.01
1  8.61e+05  782.15 |
1  8.61e+05  782.15 |  782.15   41.1    136    12 |    1.40   0.17  27.39   0.02
1  8.78e+05  782.15 |  769.69    0.0    134     0 |    1.35   0.17  29.88   0.02
1  8.95e+05  782.15 |  773.94    0.0    137     0 |    1.40   0.17  28.86   0.02
1  9.12e+05  782.15 |  579.52  159.8     95    23 |    1.37   0.17  29.90   0.02
1  9.29e+05  782.15 |  368.24    0.0     64     0 |    1.38   0.17  29.71   0.02
1  9.46e+05  782.15 |  471.58    0.0     82     0 |    1.43   0.17  29.84   0.02
1  9.63e+05  977.21 |
1  9.63e+05  977.21 |  977.21  270.7    162    42 |    1.42   0.18  30.33   0.02
1  9.81e+05  977.21 |  856.29    0.0    162     0 |    1.45   0.18  30.16   0.02
1  9.98e+05  977.21 |  594.79    0.0     99     0 |    1.42   0.17  30.40   0.02
1  1.02e+06  977.21 |  729.72    0.0    130     0 |    1.46   0.17  30.52   0.02
1  1.03e+06  977.21 |  798.60    0.0    151     0 |    1.46   0.18  30.56   0.02
1  1.05e+06 1243.32 |
1  1.05e+06 1243.32 | 1243.32  129.2    213    27 |    1.43   0.18  31.40   0.02
1  1.07e+06 1243.32 | 1213.14    0.0    199     0 |    1.49   0.18  32.20   0.02
1  1.09e+06 1243.32 |  752.98    0.0    139     0 |    1.47   0.18  31.79   0.02
1  1.10e+06 1243.32 |  788.21    0.0    148     0 |    1.49   0.18  30.92   0.02
1  1.11e+06 1243.32 |  806.23    0.0    133     0 |    1.49   0.18  31.74   0.02
1  1.12e+06 1284.78 |
1  1.12e+06 1284.78 | 1284.78  212.0    207    38 |    1.48   0.18  31.87   0.02
1  1.14e+06 1284.78 |  682.50    0.0    113     0 |    1.50   0.18  32.15   0.02
1  1.15e+06 1284.78 |  468.61    0.0     81     0 |    1.51   0.18  32.10   0.02
1  1.16e+06 1284.78 |  986.22    0.0    174     0 |    1.51   0.18  32.51   0.02
1  1.18e+06 1284.78 |  933.67    0.0    172     0 |    1.54   0.18  31.68   0.02
1  1.19e+06 1284.78 | 1189.34  202.8    195    24 |    1.58   0.18  32.28   0.02
1  1.20e+06 1284.78 | 1183.09    0.0    197     0 |    1.54   0.18  33.36   0.02
1  1.22e+06 1284.78 | 1135.20    0.0    188     0 |    1.53   0.18  31.78   0.02
1  1.23e+06 1284.78 | 1147.86    0.0    183     0 |    1.61   0.19  34.01   0.02
1  1.25e+06 1284.78 | 1251.78    0.0    225     0 |    1.56   0.18  34.35   0.02
1  1.26e+06 2821.26 |
1  1.26e+06 2821.26 | 2821.26  412.2    429    55 |    1.55   0.18  33.30   0.02
1  1.27e+06 2821.26 | 1078.30    0.0    171     0 |    1.59   0.18  33.66   0.02
1  1.29e+06 2821.26 | 2256.06 1179.8    350   166 |    1.61   0.18  33.90   0.02
1  1.30e+06 2821.26 | 1868.22    0.0    305     0 |    1.62   0.18  34.38   0.02
1  1.32e+06 2821.26 | 2667.33 1740.0    396   247 |    1.59   0.18  34.13   0.02
1  1.33e+06 2821.26 | 1578.01    0.0    244     0 |    1.60   0.18  34.10   0.02
1  1.34e+06 2821.26 | 2315.74  907.4    361   142 |    1.64   0.19  34.61   0.02
1  1.36e+06 2821.26 | 1068.61    0.0    185     0 |    1.65   0.18  34.59   0.02
1  1.37e+06 2821.26 |  780.38    0.0    119     0 |    1.68   0.19  35.22   0.02
1  1.38e+06 2821.26 | 2128.08  909.8    340   138 |    1.68   0.19  34.24   0.02
1  1.40e+06 5116.59 |
1  1.40e+06 5116.59 | 5116.59 2166.1    734   297 |    1.77   0.19  32.93   0.02
1  1.41e+06 5116.59 | 2901.13 2543.3    403   345 |    1.71   0.19  35.32   0.02
1  1.43e+06 5116.59 | 1638.18    0.0    244     0 |    1.73   0.19  34.84   0.02
1  1.44e+06 5116.59 | 1007.29    0.0    169     0 |    1.72   0.18  34.59   0.02
1  1.46e+06 5116.59 | 3304.17 2510.8    469   341 |    1.76   0.19  35.45   0.02
1  1.47e+06 5116.59 | 3501.58    0.0    483     0 |    1.74   0.19  35.97   0.02
1  1.49e+06 5305.27 |
1  1.49e+06 5305.27 | 5305.27 2807.9    730   378 |    1.77   0.19  35.58   0.02
1  1.50e+06 5305.27 | 5014.42 2359.4    688   315 |    1.72   0.19  35.65   0.02
1  1.52e+06 5305.27 | 4736.14    0.0    639     0 |    1.72   0.19  36.84   0.02
1  1.53e+06 5305.27 | 4393.27 2047.5    598   265 |    1.76   0.19  37.57   0.02
1  1.55e+06 5305.27 | 1213.84    0.0    175     0 |    1.84   0.19  37.64   0.02
1  1.56e+06 5305.27 | 3804.35 2176.2    529   286 |    1.82   0.18  36.11   0.02
1  1.58e+06 5305.27 | 4753.45    0.0    649     0 |    1.82   0.19  35.78   0.02
1  1.59e+06 5305.27 | 3659.95    0.0    478     0 |    1.81   0.20  36.94   0.02
1  1.61e+06 5305.27 | 4564.50    0.0    606     0 |    1.85   0.19  36.93   0.02
1  1.62e+06 6495.10 |
1  1.62e+06 6495.10 | 6495.10 2253.1    837   283 |    1.83   0.19  37.61   0.02
1  1.63e+06 6495.10 | 6440.03 2628.1    819   314 |    1.85   0.19  37.76   0.02
1  1.64e+06 6495.10 | 5980.08 2214.3    784   282 |    1.91   0.19  37.79   0.02
1  1.66e+06 6495.10 | 1143.72    0.0    158     0 |    1.87   0.19  38.26   0.02
1  1.67e+06 6495.10 | 3229.25    0.0    447     0 |    1.87   0.19  38.75   0.02
1  1.68e+06 7820.89 |
1  1.68e+06 7820.89 | 7820.89   49.0   1000     0 |    1.86   0.19  38.04   0.02
1  1.69e+06 7820.89 | 6616.72 2186.4    843   272 |    1.89   0.19  38.41   0.02
1  1.70e+06 7820.89 | 2205.00    0.0    297     0 |    1.85   0.19  38.45   0.02
1  1.71e+06 8201.97 |
1  1.71e+06 8201.97 | 8201.97   60.7   1000     0 |    1.89   0.19  38.49   0.02
1  1.72e+06 8201.97 | 8053.00    0.0   1000     0 |    1.93   0.19  37.79   0.02
1  1.73e+06 8201.97 | 7974.46    0.0   1000     0 |    1.92   0.19  38.88   0.02
1  1.74e+06 8201.97 | 1736.69    0.0    237     0 |    1.88   0.19  38.00   0.02
1  1.75e+06 8201.97 | 7027.53 1871.2    873   220 |    1.94   0.19  37.89   0.02
1  1.76e+06 8201.97 | 4689.30    0.0    579     0 |    1.92   0.19  38.55   0.02
1  1.77e+06 8201.97 | 7855.02    0.0   1000     0 |    1.91   0.19  38.86   0.02
1  1.78e+06 8201.97 | 5215.32    0.0    673     0 |    1.92   0.19  38.98   0.02
1  1.79e+06 8201.97 | 8026.47    0.0   1000     0 |    1.92   0.19  39.59   0.02
1  1.80e+06 8201.97 | 2568.57    0.0    322     0 |    1.89   0.19  39.10   0.02
1  1.81e+06 8201.97 | 3905.44    0.0    516     0 |    1.91   0.20  39.09   0.02
1  1.82e+06 8201.97 | 6868.42    0.0    841     0 |    1.89   0.19  38.71   0.02
1  1.83e+06 8201.97 | 2328.69    0.0    302     0 |    1.93   0.19  39.10   0.02
1  1.84e+06 8201.97 | 7570.05 1112.7    922   135 |    1.97   0.19  39.31   0.02
1  1.85e+06 8201.97 | 5452.10 3047.4    648   353 |    1.81   0.19  39.20   0.02
1  1.85e+06 8201.97 | 5185.33 3187.5    621   365 |    1.95   0.19  39.28   0.02
1  1.87e+06 8201.97 | 3431.44    0.0    414     0 |    2.01   0.19  39.72   0.02
1  1.88e+06 8201.97 | 7174.34 2007.4    852   236 |    1.92   0.20  40.53   0.02
1  1.88e+06 8201.97 | 7000.72 2664.7    823   307 |    1.98   0.19  39.93   0.02
1  1.89e+06 8201.97 | 7756.40 1533.1    900   172 |    1.95   0.20  40.53   0.02
1  1.90e+06 8201.97 | 1015.23    0.0    138     0 |    1.97   0.20  40.52   0.02
1  1.91e+06 8201.97 | 7223.27    0.0    829     0 |    1.96   0.20  39.91   0.02
1  1.92e+06 8201.97 | 2276.04    0.0    271     0 |    1.93   0.20  41.08   0.02
1  1.93e+06 8201.97 | 4279.54 2531.4    512   290 |    1.92   0.20  41.14   0.02
1  1.94e+06 8601.43 |
1  1.94e+06 8601.43 | 8601.43   36.4   1000     0 |    2.01   0.19  41.16   0.02
1  1.95e+06 8605.06 |
1  1.95e+06 8605.06 | 8605.06   53.1    999     2 |    2.01   0.19  39.77   0.02
1  1.96e+06 8605.06 | 8604.10    0.0   1000     0 |    2.02   0.19  41.39   0.02
1  1.97e+06 8605.06 | 4942.93 2670.4    565   286 |    1.99   0.19  40.93   0.02
1  1.98e+06 8605.06 | 8122.71 1366.2    914   148 |    1.93   0.20  41.49   0.02
1  1.99e+06 8605.06 | 1545.50    0.0    194     0 |    1.96   0.19  40.50   0.02
1  2.00e+06 8605.06 | 8586.34    0.0   1000     0 |    1.98   0.20  40.69   0.02
| UsedTime:   17168 | SavedDir: ./Humanoid-v3_ReliableSAC_1
| Learner: Save in ./Humanoid-v3_ReliableSAC_1

        """
    elif env_name == 'Humanoid-v3.backup.best.5684':
        from elegantrl.envs.CustomGymEnv import HumanoidEnv
        env_func = HumanoidEnv
        env_args = {
            'env_num': 1,
            'env_name': 'Humanoid-v3',
            'max_step': 1000,
            'state_dim': 376,
            'action_dim': 17,
            'if_discrete': False,
            'target_return': 3000.,
        }
        args = Arguments(agent, env_func=env_func, env_args=env_args)

        args.eval_times = 2 ** 2
        args.reward_scale = 2 ** -5  # todo
        args.max_memo = 2 ** 21
        args.learning_rate = 2 ** -14
        args.lambda_a_log_std = 2 ** -6

        args.target_step = args.max_step
        args.worker_num = 4

        args.net_dim = 2 ** 8
        args.num_layer = 4
        args.batch_size = args.net_dim
        args.repeat_times = 2 ** 1
        args.gamma = 0.99
        args.if_act_target = False  # todo
        # import numpy as np
        # args.target_entropy = np.log(env_args['action_dim']) * 1.5  # ???

        args.if_allow_break = False
        args.break_step = int(2e6)

        """
        | Arguments Remove cwd: ./Humanoid-v3_ReliableSAC_2
        ################################################################################
        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        2  8.07e+03   61.53 |
        2  8.07e+03   61.53 |   61.53    0.2     13     0 |    0.30   0.54   0.06   0.00
        2  8.94e+04  280.97 |
        2  8.94e+04  280.97 |  280.97   31.6     65     7 |    0.27   0.07   7.21   0.00
        2  1.30e+05  482.54 |
        2  1.30e+05  482.54 |  482.54   81.2    104    18 |    0.29   0.14  12.59   0.00
        2  1.63e+05  482.54 |  214.11    0.0     45     0 |    0.33   0.19  14.43   0.00
        2  1.92e+05  482.54 |  316.52    0.0     68     0 |    0.28   0.21  14.28   0.00
        2  2.17e+05  482.54 |  420.99   62.4     85    13 |    0.32   0.21  14.13   0.01
        2  2.37e+05  482.54 |  431.52   86.9     93    20 |    0.31   0.17  12.33   0.01
        2  2.58e+05  482.54 |  327.86    0.0     66     0 |    0.31   0.16  11.73   0.02
        2  2.79e+05  482.54 |  242.40    0.0     52     0 |    0.29   0.15  11.19   0.02
        2  2.96e+05  482.54 |  320.37    0.0     69     0 |    0.30   0.14  11.86   0.02
        2  3.13e+05  482.54 |  322.18    0.0     63     0 |    0.31   0.13  11.78   0.02
        2  3.30e+05  482.54 |  329.25    0.0     65     0 |    0.30   0.13  12.85   0.02
        2  3.46e+05  482.54 |  325.96    0.0     64     0 |    0.31   0.13  13.95   0.02
        2  3.63e+05  482.54 |  404.71    0.0     86     0 |    0.30   0.13  13.56   0.02
        2  3.76e+05  482.54 |  470.03    0.0     90     0 |    0.31   0.14  13.65   0.02
        2  3.88e+05  482.54 |  476.55    0.0    105     0 |    0.31   0.14  14.21   0.02
        2  4.01e+05  482.54 |  442.83  129.8     88    23 |    0.31   0.14  14.80   0.02
        2  4.13e+05  634.66 |
        2  4.13e+05  634.66 |  634.66  120.2    129    21 |    0.30   0.15  13.76   0.02
        2  4.26e+05  634.66 |  559.63   50.0    121    16 |    0.30   0.15  15.50   0.02
        2  4.39e+05  634.66 |  523.28  170.8    112    31 |    0.29   0.15  15.04   0.02
        2  4.51e+05  634.66 |  357.70    0.0     77     0 |    0.30   0.16  15.47   0.02
        2  4.64e+05  634.66 |  324.52    0.0     73     0 |    0.30   0.17  15.74   0.02
        2  4.77e+05  634.66 |  605.87  211.2    128    35 |    0.31   0.18  15.78   0.02
        2  4.90e+05  634.66 |  525.32    0.0    102     0 |    0.30   0.18  16.98   0.02
        2  5.03e+05  634.66 |  606.06    0.0    119     0 |    0.30   0.20  17.79   0.02
        2  5.16e+05  754.18 |
        2  5.16e+05  754.18 |  754.18  187.2    152    40 |    0.30   0.20  18.10   0.02
        2  5.28e+05  754.18 |  588.85    0.0    121     0 |    0.30   0.20  18.44   0.02
        2  5.41e+05  754.18 |  645.80    0.0    137     0 |    0.29   0.21  18.65   0.02
        2  5.54e+05  754.18 |  599.06  180.4    127    37 |    0.29   0.21  17.64   0.02
        2  5.68e+05  754.18 |  598.60  125.8    116    27 |    0.30   0.22  18.42   0.02
        2  5.80e+05  754.18 |  648.31  153.8    139    28 |    0.29   0.22  21.32   0.02
        2  5.93e+05  754.18 |  687.98    0.0    154     0 |    0.30   0.23  19.56   0.02
        2  6.02e+05  754.18 |  433.99    0.0     95     0 |    0.30   0.23  20.28   0.02
        2  6.11e+05  754.18 |  592.46  233.4    127    41 |    0.30   0.23  21.65   0.02
        2  6.19e+05  754.18 |  443.31    0.0     89     0 |    0.30   0.23  20.45   0.02
        2  6.28e+05  754.18 |  630.86  127.6    124    24 |    0.30   0.23  21.10   0.02
        2  6.37e+05  844.03 |
        2  6.37e+05  844.03 |  844.03  237.1    164    45 |    0.31   0.24  21.83   0.02
        2  6.46e+05  995.82 |
        2  6.46e+05  995.82 |  995.82  296.4    204    53 |    0.31   0.24  21.79   0.02
        2  6.55e+05  995.82 |  704.54    0.0    146     0 |    0.31   0.25  22.99   0.02
        2  6.64e+05  995.82 |  597.17    0.0    120     0 |    0.31   0.25  23.18   0.02
        2  6.73e+05  995.82 |  578.13    0.0    127     0 |    0.31   0.26  23.02   0.02
        2  6.82e+05  995.82 |  985.69    0.0    188     0 |    0.30   0.26  22.20   0.02
        2  6.90e+05 1329.88 |
        2  6.90e+05 1329.88 | 1329.88  453.5    271    86 |    0.30   0.26  21.81   0.02
        2  6.99e+05 1329.88 |  984.74  242.2    202    52 |    0.30   0.26  24.70   0.02
        2  7.08e+05 1329.88 | 1282.09  424.1    252    80 |    0.31   0.26  24.36   0.02
        2  7.17e+05 1609.77 |
        2  7.17e+05 1609.77 | 1609.77  986.8    321   199 |    0.30   0.26  25.79   0.02
        2  7.26e+05 1835.22 |
        2  7.26e+05 1835.22 | 1835.22  846.6    361   168 |    0.31   0.26  22.76   0.02
        2  7.35e+05 1835.22 | 1455.07    0.0    301     0 |    0.31   0.27  24.73   0.02
        2  7.45e+05 1835.22 | 1424.00    0.0    284     0 |    0.31   0.26  25.26   0.02
        2  7.54e+05 1835.22 | 1585.16  471.9    319    96 |    0.31   0.26  24.13   0.02
        2  7.63e+05 1835.22 |  502.12    0.0     99     0 |    0.30   0.27  24.48   0.02
        2  7.72e+05 1957.16 |
        2  7.72e+05 1957.16 | 1957.16 1026.3    374   191 |    0.31   0.27  25.86   0.02
        2  7.81e+05 1957.16 |  518.58    0.0    106     0 |    0.31   0.27  25.86   0.02
        2  7.91e+05 1957.16 | 1099.69    0.0    223     0 |    0.31   0.27  25.92   0.02
        2  8.01e+05 1957.16 | 1239.12    0.0    246     0 |    0.31   0.27  26.76   0.02
        2  8.10e+05 1957.16 | 1706.44    0.0    318     0 |    0.31   0.27  25.48   0.02
        2  8.20e+05 1957.16 | 1875.97  612.2    369   110 |    0.31   0.27  26.70   0.02
        2  8.30e+05 1957.16 |  797.77    0.0    158     0 |    0.31   0.28  26.81   0.02
        2  8.39e+05 1957.16 | 1699.40    0.0    339     0 |    0.31   0.27  26.47   0.02
        2  8.48e+05 1957.16 |  854.85    0.0    166     0 |    0.32   0.28  28.31   0.02
        2  8.58e+05 1957.16 | 1492.19    0.0    290     0 |    0.32   0.28  25.77   0.02
        2  8.67e+05 1957.16 | 1687.29    0.0    332     0 |    0.31   0.28  26.35   0.02
        2  8.76e+05 2319.36 |
        2  8.76e+05 2319.36 | 2319.36 1121.7    436   208 |    0.31   0.28  27.42   0.02
        2  8.86e+05 2319.36 | 1523.18    0.0    287     0 |    0.32   0.28  27.89   0.02
        2  8.95e+05 3929.77 |
        2  8.95e+05 3929.77 | 3929.77 1606.8    757   308 |    0.31   0.28  28.16   0.02
        2  9.05e+05 3929.77 | 1623.65    0.0    312     0 |    0.31   0.28  27.18   0.02
        2  9.15e+05 3929.77 | 3908.78    0.0    752     0 |    0.32   0.28  28.58   0.02
        2  9.25e+05 3929.77 | 1302.67    0.0    258     0 |    0.32   0.27  29.20   0.02
        2  9.35e+05 3929.77 | 1261.39    0.0    245     0 |    0.32   0.27  27.75   0.02
        2  9.45e+05 3929.77 |  759.79    0.0    169     0 |    0.31   0.27  28.00   0.02
        2  9.55e+05 3929.77 | 2116.41    0.0    402     0 |    0.31   0.26  28.82   0.02
        2  9.65e+05 3929.77 | 1699.71    0.0    315     0 |    0.32   0.27  30.17   0.02
        2  9.74e+05 3929.77 |  726.84    0.0    160     0 |    0.32   0.27  28.64   0.02
        2  9.84e+05 4231.71 |
        2  9.84e+05 4231.71 | 4231.71 1425.9    799   257 |    0.32   0.27  28.88   0.02
        2  9.94e+05 4231.71 | 2501.40 1341.9    474   253 |    0.31   0.27  30.48   0.02
        2  1.00e+06 4231.71 | 3515.81 1836.7    660   333 |    0.32   0.27  30.17   0.02
        2  1.01e+06 4231.71 |  828.37    0.0    171     0 |    0.33   0.27  30.49   0.02
        2  1.02e+06 4231.71 | 1462.77    0.0    270     0 |    0.32   0.27  31.05   0.02
        2  1.03e+06 4231.71 | 1928.92    0.0    356     0 |    0.33   0.26  31.43   0.02
        2  1.04e+06 4231.71 |  609.80    0.0    128     0 |    0.32   0.26  30.16   0.02
        2  1.05e+06 4231.71 | 3092.45 1241.4    596   244 |    0.32   0.27  31.47   0.02
        2  1.06e+06 4231.71 | 1380.05    0.0    261     0 |    0.32   0.25  32.00   0.02
        2  1.07e+06 4231.71 | 2293.69    0.0    423     0 |    0.32   0.26  32.25   0.02
        2  1.08e+06 4231.71 |  904.86    0.0    180     0 |    0.32   0.25  32.23   0.02
        2  1.10e+06 4231.71 | 1727.42    0.0    351     0 |    0.32   0.25  31.05   0.02
        2  1.11e+06 4231.71 | 2375.34    0.0    441     0 |    0.33   0.25  30.80   0.02
        2  1.12e+06 4231.71 | 3723.47    0.0    684     0 |    0.33   0.25  32.45   0.02
        2  1.13e+06 4231.71 | 4135.63 1886.7    768   342 |    0.32   0.25  31.90   0.02
        2  1.14e+06 4231.71 |  765.19    0.0    142     0 |    0.33   0.24  32.01   0.02
        2  1.15e+06 4231.71 | 1945.57    0.0    366     0 |    0.33   0.24  34.65   0.02
        2  1.16e+06 4231.71 | 2542.68 1734.2    465   311 |    0.32   0.24  32.80   0.02
        2  1.17e+06 4231.71 | 2590.14    0.0    471     0 |    0.32   0.24  31.36   0.02
        2  1.18e+06 4231.71 | 2468.01    0.0    447     0 |    0.33   0.25  33.19   0.02
        2  1.19e+06 4231.71 | 1560.16    0.0    285     0 |    0.33   0.24  31.76   0.02
        2  1.19e+06 4231.71 |  223.32    0.0     43     0 |    0.33   0.24  34.08   0.02
        2  1.20e+06 4231.71 |  123.22    0.0     25     0 |    0.33   0.25  32.51   0.02
        2  1.21e+06 4231.71 | 1063.34    0.0    205     0 |    0.32   0.24  33.16   0.02
        2  1.21e+06 4231.71 | 2342.22 2161.3    434   392 |    0.32   0.24  32.86   0.02
        2  1.22e+06 4231.71 | 1376.66    0.0    254     0 |    0.33   0.24  33.61   0.02
        2  1.22e+06 4231.71 | 1939.27    0.0    345     0 |    0.33   0.24  32.21   0.02
        2  1.23e+06 4231.71 | 1450.92    0.0    282     0 |    0.33   0.23  34.04   0.02
        2  1.23e+06 4231.71 | 3652.45    0.0    657     0 |    0.34   0.24  34.21   0.02
        2  1.24e+06 4231.71 | 1252.34    0.0    240     0 |    0.33   0.24  33.65   0.02
        2  1.24e+06 4231.71 | 1768.35    0.0    326     0 |    0.34   0.24  33.81   0.02
        2  1.25e+06 4231.71 | 1952.18    0.0    350     0 |    0.33   0.24  34.21   0.02
        2  1.25e+06 4231.71 | 1479.76    0.0    278     0 |    0.33   0.23  34.37   0.02
        2  1.26e+06 4231.71 | 4231.34    0.0    757     0 |    0.34   0.24  33.61   0.02
        2  1.26e+06 4231.71 | 4202.22    0.0    755     0 |    0.33   0.23  32.39   0.02
        2  1.27e+06 4231.71 | 3597.61 2100.9    658   379 |    0.33   0.24  32.62   0.02
        2  1.27e+06 4231.71 |  430.31    0.0     92     0 |    0.33   0.23  34.13   0.02
        2  1.28e+06 4231.71 | 2833.13 1841.9    511   320 |    0.33   0.23  34.23   0.01
        2  1.28e+06 4351.07 |
        2  1.28e+06 4351.07 | 4351.07 1734.1    778   304 |    0.33   0.23  32.08   0.01
        2  1.29e+06 4351.07 | 2608.48    0.0    470     0 |    0.33   0.24  34.04   0.01
        2  1.29e+06 4351.07 | 2364.01 1791.9    428   311 |    0.33   0.23  34.24   0.01
        2  1.30e+06 4351.07 | 3731.13 1500.9    686   270 |    0.33   0.23  35.38   0.01
        2  1.30e+06 4351.07 |  805.48    0.0    158     0 |    0.33   0.23  31.95   0.01
        2  1.31e+06 4351.07 |  491.94    0.0    109     0 |    0.33   0.23  33.86   0.01
        2  1.31e+06 4351.07 | 1352.71    0.0    244     0 |    0.33   0.22  33.14   0.01
        2  1.32e+06 4351.07 | 1908.86    0.0    352     0 |    0.33   0.23  32.84   0.01
        2  1.32e+06 4351.07 |  950.31    0.0    171     0 |    0.33   0.22  34.09   0.01
        2  1.33e+06 4351.07 |  639.88    0.0    124     0 |    0.33   0.22  32.27   0.01
        2  1.34e+06 4351.07 | 2889.20    0.0    515     0 |    0.33   0.22  34.52   0.01
        2  1.34e+06 4351.07 | 3991.38    0.0    768     0 |    0.33   0.22  33.87   0.01
        2  1.35e+06 4351.07 | 3834.92    0.0    699     0 |    0.34   0.23  35.08   0.01
        2  1.35e+06 4351.07 | 1375.02    0.0    253     0 |    0.33   0.23  33.69   0.01
        2  1.35e+06 4351.07 | 3717.74    0.0    671     0 |    0.33   0.23  31.91   0.01
        2  1.36e+06 4351.07 | 3654.43 1926.9    668   345 |    0.32   0.22  33.76   0.01
        2  1.36e+06 4351.07 | 4149.08 1117.4    737   191 |    0.33   0.22  33.57   0.01
        2  1.37e+06 4584.25 |
        2  1.37e+06 4584.25 | 4584.25 1056.6    819   187 |    0.33   0.22  33.02   0.01
        2  1.38e+06 4584.25 |  745.19    0.0    149     0 |    0.34   0.23  34.83   0.01
        2  1.38e+06 4584.25 |  764.75    0.0    143     0 |    0.33   0.22  33.36   0.01
        2  1.38e+06 4584.25 | 1127.75    0.0    221     0 |    0.33   0.22  34.09   0.01
        2  1.39e+06 4584.25 | 1979.66    0.0    355     0 |    0.34   0.21  34.31   0.01
        2  1.39e+06 4584.25 | 1755.43    0.0    343     0 |    0.34   0.22  34.75   0.01
        2  1.40e+06 4584.25 |  332.25    0.0     63     0 |    0.34   0.22  35.14   0.01
        2  1.40e+06 4584.25 | 1491.16    0.0    266     0 |    0.33   0.22  34.39   0.01
        2  1.41e+06 4980.79 |
        2  1.41e+06 4980.79 | 4980.79  558.8    895   105 |    0.34   0.21  34.81   0.01
        2  1.41e+06 4980.79 | 1351.99    0.0    245     0 |    0.34   0.21  34.57   0.01
        2  1.42e+06 5684.42 |
        2  1.42e+06 5684.42 | 5684.42  131.2   1000     0 |    0.33   0.21  33.68   0.01
        2  1.42e+06 5684.42 | 4778.06    0.0    879     0 |    0.33   0.21  33.75   0.01
        2  1.43e+06 5684.42 | 3606.03 2215.2    640   386 |    0.34   0.20  33.83   0.01
        2  1.43e+06 5684.42 | 1853.79    0.0    332     0 |    0.33   0.21  34.58   0.01
        2  1.44e+06 5684.42 | 1923.59    0.0    357     0 |    0.34   0.21  33.56   0.01
        2  1.45e+06 5684.42 | 3439.89    0.0    628     0 |    0.35   0.21  35.52   0.01
        2  1.45e+06 5684.42 | 1380.56    0.0    271     0 |    0.33   0.21  34.46   0.01
        2  1.46e+06 5684.42 | 4449.71 1530.1    798   258 |    0.33   0.21  36.05   0.01
        2  1.46e+06 5684.42 | 1432.35    0.0    283     0 |    0.34   0.21  34.44   0.01
        2  1.47e+06 5684.42 | 1907.75    0.0    343     0 |    0.34   0.21  34.38   0.01
        2  1.47e+06 5684.42 | 1346.14    0.0    251     0 |    0.32   0.21  34.43   0.01
        2  1.48e+06 5684.42 | 4292.50    0.0    802     0 |    0.34   0.21  35.61   0.01
        2  1.48e+06 5684.42 | 4377.58    0.0    768     0 |    0.34   0.21  35.64   0.01
        2  1.49e+06 5684.42 |  814.67    0.0    147     0 |    0.34   0.21  34.39   0.01
        2  1.49e+06 5684.42 | 3326.43    0.0    585     0 |    0.33   0.21  35.33   0.01
        2  1.50e+06 5684.42 | 1169.90    0.0    221     0 |    0.32   0.22  34.27   0.01
        2  1.50e+06 5684.42 |  352.07    0.0     74     0 |    0.33   0.25  35.59   0.02
        2  1.51e+06 5684.42 |  432.39    0.0     84     0 |    0.32   0.26  37.55   0.02
        2  1.51e+06 5684.42 | 3580.22    0.0    677     0 |    0.31   0.25  38.36   0.01
        2  1.52e+06 5684.42 | 3384.96    0.0    586     0 |    0.31   0.24  36.76   0.01
        2  1.52e+06 5684.42 | 1773.22    0.0    361     0 |    0.32   0.23  36.11   0.01
        2  1.53e+06 5684.42 | 2135.71    0.0    398     0 |    0.35   0.22  35.34   0.01
        2  1.53e+06 5684.42 |  860.46    0.0    156     0 |    0.34   0.22  35.35   0.01
        2  1.54e+06 5684.42 | 5422.32    0.0   1000     0 |    0.34   0.22  36.33   0.01
        2  1.54e+06 5684.42 |  675.59    0.0    127     0 |    0.34   0.21  35.75   0.01
        2  1.55e+06 5684.42 | 1234.09    0.0    222     0 |    0.33   0.21  34.51   0.01
        2  1.55e+06 5684.42 | 4229.33    0.0    753     0 |    0.34   0.20  35.09   0.01
        2  1.56e+06 5684.42 | 1366.60    0.0    244     0 |    0.33   0.20  33.57   0.01
        2  1.56e+06 5684.42 | 5525.51    0.0   1000     0 |    0.33   0.21  35.46   0.01
        2  1.57e+06 5684.42 | 1769.33    0.0    339     0 |    0.34   0.20  34.62   0.01
        2  1.57e+06 5684.42 | 2902.24    0.0    546     0 |    0.34   0.20  34.76   0.01
        2  1.58e+06 5684.42 | 4058.17    0.0    721     0 |    0.34   0.21  34.88   0.01
        2  1.58e+06 5684.42 | 5075.46 1003.3    900   173 |    0.34   0.21  34.74   0.01
        2  1.59e+06 5684.42 | 3907.85    0.0    694     0 |    0.34   0.21  36.51   0.01
        2  1.59e+06 5684.42 | 5449.51    0.0   1000     0 |    0.34   0.20  34.68   0.01
        2  1.60e+06 5684.42 | 4781.07    0.0    863     0 |    0.34   0.21  35.13   0.01
        2  1.61e+06 5684.42 |  797.45    0.0    151     0 |    0.34   0.20  35.76   0.01
        2  1.61e+06 5684.42 | 1150.24    0.0    217     0 |    0.33   0.20  35.66   0.01
        2  1.62e+06 5684.42 | 5660.03    0.0   1000     0 |    0.34   0.20  35.80   0.01
        2  1.62e+06 5684.42 | 3585.73 1437.3    617   237 |    0.35   0.20  34.86   0.01
        2  1.63e+06 5684.42 | 4281.01 1572.7    744   258 |    0.33   0.19  35.23   0.01
        2  1.63e+06 5684.42 | 2157.23    0.0    399     0 |    0.34   0.20  34.19   0.01
        2  1.64e+06 5684.42 | 5280.96    0.0   1000     0 |    0.35   0.19  33.86   0.01
        2  1.64e+06 5684.42 | 5135.08    0.0    871     0 |    0.34   0.20  34.85   0.01
        2  1.65e+06 5684.42 | 1028.12    0.0    187     0 |    0.34   0.19  35.61   0.01
        2  1.65e+06 5684.42 | 1541.23    0.0    272     0 |    0.35   0.19  35.61   0.01
        2  1.66e+06 5684.42 | 2266.63    0.0    416     0 |    0.34   0.19  33.94   0.01
        2  1.66e+06 5684.42 | 1899.89    0.0    342     0 |    0.35   0.19  34.28   0.01
        2  1.67e+06 5684.42 | 4176.02 1000.3    717   164 |    0.34   0.19  33.48   0.01
        2  1.67e+06 5684.42 | 5189.92 1227.2    882   205 |    0.34   0.19  35.69   0.01
        2  1.68e+06 5684.42 |  195.31    0.0     38     0 |    0.35   0.19  34.47   0.01
        2  1.68e+06 5684.42 | 5663.44    0.0   1000     0 |    0.35   0.19  36.95   0.01
        2  1.69e+06 5684.42 |  823.29    0.0    153     0 |    0.34   0.19  35.86   0.01
        2  1.69e+06 5684.42 | 3310.77    0.0    587     0 |    0.34   0.19  35.26   0.01
        2  1.70e+06 5684.42 | 5610.64    0.0   1000     0 |    0.34   0.19  36.32   0.01
        2  1.70e+06 5684.42 | 2492.52    0.0    441     0 |    0.35   0.20  35.55   0.01
        2  1.71e+06 5684.42 | 1382.81    0.0    251     0 |    0.34   0.19  35.50   0.01
        2  1.71e+06 5684.42 | 5502.84    0.0    965     0 |    0.34   0.19  35.44   0.01
        2  1.72e+06 5684.42 | 2653.70    0.0    464     0 |    0.34   0.19  35.49   0.01
        2  1.72e+06 5684.42 | 2569.08    0.0    446     0 |    0.34   0.20  34.85   0.01
        2  1.73e+06 5684.42 | 1960.59    0.0    356     0 |    0.35   0.20  35.89   0.01
        2  1.73e+06 5684.42 |  569.35    0.0    122     0 |    0.35   0.20  35.76   0.01
        2  1.74e+06 5684.42 | 1555.08    0.0    302     0 |    0.33   0.19  34.31   0.01
        2  1.74e+06 5684.42 |  825.29    0.0    181     0 |    0.33   0.20  35.71   0.01
        2  1.75e+06 5684.42 |  884.30    0.0    182     0 |    0.33   0.20  35.93   0.01
        2  1.75e+06 5684.42 |  754.74    0.0    154     0 |    0.33   0.20  35.86   0.01
        2  1.76e+06 5684.42 |  840.29    0.0    166     0 |    0.32   0.19  35.76   0.01
        2  1.76e+06 5684.42 | 5375.29    0.0    927     0 |    0.32   0.18  35.98   0.01
        2  1.76e+06 5684.42 | 2216.25    0.0    376     0 |    0.34   0.19  35.01   0.01
        2  1.77e+06 5684.42 | 5582.28    0.0   1000     0 |    0.34   0.19  36.68   0.01
        2  1.77e+06 5684.42 | 3306.34    0.0    592     0 |    0.35   0.18  36.32   0.01
        2  1.78e+06 5684.42 | 5509.94    0.0    957     0 |    0.34   0.19  35.58   0.01
        2  1.78e+06 5684.42 | 1581.59    0.0    275     0 |    0.34   0.19  34.64   0.01
        2  1.79e+06 5684.42 | 4884.17 1757.6    831   292 |    0.34   0.19  34.52   0.01
        2  1.79e+06 5684.42 |  520.76    0.0    102     0 |    0.34   0.19  36.42   0.01
        2  1.80e+06 5684.42 | 2075.62    0.0    360     0 |    0.36   0.19  34.56   0.01
        2  1.80e+06 5684.42 | 2849.00    0.0    504     0 |    0.35   0.19  36.11   0.01

        """
    elif env_name == 'Humanoid-v3.backup.5606':
        from elegantrl.envs.CustomGymEnv import HumanoidEnv
        env_func = HumanoidEnv
        env_args = {
            'env_num': 1,
            'env_name': 'Humanoid-v3',
            'max_step': 1000,
            'state_dim': 376,
            'action_dim': 17,
            'if_discrete': False,
            'target_return': 3000.,
        }
        args = Arguments(agent, env_func=env_func, env_args=env_args)

        args.eval_times = 2 ** 2
        args.reward_scale = 2 ** -4
        args.max_memo = 2 ** 21
        args.learning_rate = 2 ** -14
        args.lambda_a_log_std = 2 ** -6

        args.target_step = args.max_step
        args.worker_num = 4

        args.net_dim = 2 ** 8
        args.num_layer = 4
        args.batch_size = args.net_dim
        args.repeat_times = 2 ** 1
        args.gamma = 0.99

        args.if_allow_break = False
        args.break_step = int(2e6)

        """
        | Arguments Remove cwd: ./Humanoid-v3_ReliableSAC_2
        ################################################################################
        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        2  8.07e+03   76.16 |
        2  8.07e+03   76.16 |   76.16    0.1     16     0 |    0.30   0.55  -0.11   0.00
        2  8.47e+04  226.51 |
        2  8.47e+04  226.51 |  226.51   12.0     45     2 |    0.29   0.03   4.31   0.00
        2  1.26e+05  330.59 |
        2  1.26e+05  330.59 |  330.59   12.2     70     2 |    0.29   0.07   8.89   0.00
        2  1.59e+05  502.37 |
        2  1.59e+05  502.37 |  502.37   74.8    102    13 |    0.33   0.08  10.22   0.00
        2  1.89e+05  502.37 |  271.74    0.0     51     0 |    0.32   0.13  13.05   0.00
        2  2.13e+05  502.37 |  263.78    0.0     58     0 |    0.32   0.16  12.73   0.01
        2  2.34e+05  502.37 |  333.15    0.0     65     0 |    0.28   0.15  13.12   0.01
        2  2.55e+05  502.37 |  480.34   70.0    105    13 |    0.32   0.13  11.47   0.02
        2  2.76e+05  538.58 |
        2  2.76e+05  538.58 |  538.58  108.7    108    19 |    0.31   0.11  12.34   0.02
        2  2.92e+05  538.58 |  357.07    0.0     68     0 |    0.30   0.12  12.55   0.02
        2  3.09e+05  538.58 |  401.06    0.0     79     0 |    0.30   0.12  12.48   0.02
        2  3.26e+05  538.58 |  359.38    0.0     78     0 |    0.31   0.13  14.05   0.02
        2  3.43e+05  538.58 |  415.86    0.0     98     0 |    0.30   0.14  14.95   0.02
        2  3.59e+05  538.58 |  427.16    0.0     80     0 |    0.31   0.15  14.31   0.02
        2  3.77e+05  538.58 |  486.99    0.0     93     0 |    0.30   0.16  14.94   0.02
        2  3.89e+05  538.58 |  462.94    0.0    101     0 |    0.30   0.17  15.75   0.02
        2  4.01e+05  538.58 |  419.89    0.0     99     0 |    0.30   0.18  15.39   0.02
        2  4.14e+05  538.58 |  479.96    0.0     90     0 |    0.31   0.19  15.36   0.02
        2  4.27e+05  538.58 |  537.61    0.0    119     0 |    0.30   0.20  15.81   0.02
        2  4.39e+05  538.58 |  363.96    0.0     75     0 |    0.30   0.21  15.95   0.02
        2  4.52e+05  538.58 |  535.53  158.0    110    35 |    0.30   0.21  17.68   0.02
        2  4.65e+05  538.58 |  434.56    0.0     90     0 |    0.31   0.22  17.65   0.02
        2  4.78e+05  538.58 |  421.31    0.0     88     0 |    0.31   0.22  16.96   0.02
        2  4.91e+05  538.58 |  372.28    0.0     74     0 |    0.30   0.23  17.25   0.02
        2  5.03e+05  538.58 |  455.47  128.7     96    29 |    0.30   0.23  18.11   0.02
        2  5.16e+05  538.58 |  428.26    0.0     86     0 |    0.31   0.24  18.75   0.02
        2  5.29e+05  538.58 |  316.51    0.0     62     0 |    0.31   0.24  19.18   0.02
        2  5.42e+05  615.80 |
        2  5.42e+05  615.80 |  615.80   96.7    123    21 |    0.31   0.25  21.45   0.02
        2  5.54e+05  826.89 |
        2  5.54e+05  826.89 |  826.89  104.2    168    26 |    0.30   0.27  20.43   0.02
        2  5.67e+05  826.89 |  450.51    0.0     92     0 |    0.31   0.27  19.76   0.02
        2  5.80e+05  826.89 |  498.25    0.0     92     0 |    0.31   0.27  21.87   0.02
        2  5.93e+05  826.89 |  815.60  147.8    166    33 |    0.31   0.27  22.05   0.02
        2  6.02e+05  826.89 |  699.31    0.0    147     0 |    0.30   0.28  22.44   0.02
        2  6.10e+05  826.89 |  643.56    0.0    126     0 |    0.31   0.28  23.44   0.02
        2  6.19e+05  826.89 |  167.29    0.0     33     0 |    0.31   0.28  23.14   0.02
        2  6.28e+05  826.89 |  367.48    0.0     74     0 |    0.31   0.29  23.11   0.02
        2  6.36e+05  826.89 |  658.05    0.0    127     0 |    0.31   0.29  22.56   0.02
        2  6.45e+05  826.89 |  746.85    0.0    147     0 |    0.31   0.29  23.29   0.02
        2  6.54e+05  826.89 |  767.08    0.0    168     0 |    0.30   0.29  21.61   0.02
        2  6.62e+05  826.89 |  736.44    0.0    139     0 |    0.30   0.30  23.38   0.02
        2  6.71e+05  826.89 |  436.42    0.0     94     0 |    0.30   0.30  22.17   0.02
        2  6.80e+05  826.89 |  634.87    0.0    120     0 |    0.30   0.29  24.19   0.02
        2  6.89e+05  826.89 |  595.90  189.0    123    44 |    0.31   0.29  26.35   0.02
        2  6.97e+05  826.89 |  567.10    0.0    107     0 |    0.31   0.31  24.67   0.02
        2  7.06e+05  826.89 |  595.54    0.0    114     0 |    0.31   0.31  24.35   0.02
        2  7.15e+05  826.89 |  399.45    0.0     77     0 |    0.31   0.30  23.58   0.02
        2  7.23e+05  826.89 |  434.58    0.0     85     0 |    0.31   0.31  25.30   0.02
        2  7.32e+05  826.89 |  446.36    0.0     83     0 |    0.31   0.31  26.40   0.02
        2  7.40e+05  826.89 |  537.46    0.0     97     0 |    0.31   0.31  24.95   0.02
        2  7.49e+05  826.89 |  671.12    0.0    137     0 |    0.32   0.30  24.95   0.02
        2  7.58e+05  826.89 |  453.14    0.0     87     0 |    0.31   0.31  25.86   0.02
        2  7.67e+05  826.89 |  691.16    0.0    130     0 |    0.31   0.30  25.07   0.02
        2  7.75e+05 1530.83 |
        2  7.75e+05 1530.83 | 1530.83  396.3    297    80 |    0.30   0.31  23.88   0.02
        2  7.84e+05 1530.83 |  891.09    0.0    180     0 |    0.31   0.30  25.45   0.02
        2  7.93e+05 1530.83 |  360.46    0.0     70     0 |    0.31   0.31  26.00   0.02
        2  8.01e+05 1530.83 |  452.85    0.0     90     0 |    0.30   0.30  24.52   0.02
        2  8.10e+05 1530.83 |  663.09    0.0    131     0 |    0.32   0.30  26.27   0.02
        2  8.19e+05 1530.83 | 1212.53  511.4    247   106 |    0.31   0.30  26.67   0.02
        2  8.28e+05 1530.83 |  799.14    0.0    151     0 |    0.31   0.30  23.82   0.02
        2  8.37e+05 1530.83 |  779.43    0.0    156     0 |    0.31   0.30  25.61   0.02
        2  8.45e+05 1530.83 |  631.44    0.0    118     0 |    0.31   0.30  25.03   0.02
        2  8.54e+05 1530.83 | 1248.43    0.0    248     0 |    0.31   0.30  25.75   0.02
        2  8.63e+05 1530.83 |  928.86    0.0    182     0 |    0.31   0.30  26.13   0.02
        2  8.73e+05 1530.83 | 1125.56    0.0    219     0 |    0.31   0.30  26.08   0.02
        2  8.82e+05 1530.83 |  698.18    0.0    145     0 |    0.31   0.30  26.49   0.02
        2  8.91e+05 1530.83 |  961.80    0.0    212     0 |    0.30   0.30  26.33   0.02
        2  9.00e+05 1530.83 | 1174.05    0.0    230     0 |    0.31   0.30  26.08   0.02
        2  9.09e+05 1530.83 | 1100.58    0.0    239     0 |    0.31   0.30  27.55   0.02
        2  9.18e+05 1868.20 |
        2  9.18e+05 1868.20 | 1868.20  164.8    371    30 |    0.30   0.30  26.45   0.02
        2  9.28e+05 1868.20 | 1073.36    0.0    201     0 |    0.31   0.30  26.89   0.02
        2  9.37e+05 1868.20 | 1205.66    0.0    230     0 |    0.31   0.30  26.77   0.02
        2  9.46e+05 1868.20 | 1372.51    0.0    267     0 |    0.31   0.29  28.57   0.02
        2  9.56e+05 1868.20 |  798.34    0.0    154     0 |    0.31   0.30  28.85   0.02
        2  9.66e+05 1868.20 | 1428.51  788.3    277   150 |    0.32   0.29  27.15   0.02
        2  9.75e+05 1868.20 |  608.42    0.0    116     0 |    0.31   0.28  29.42   0.02
        2  9.85e+05 1868.20 |  325.33    0.0     64     0 |    0.31   0.29  28.10   0.02
        2  9.95e+05 1928.41 |
        2  9.95e+05 1928.41 | 1928.41  822.3    379   159 |    0.32   0.29  29.61   0.02
        2  1.00e+06 1928.41 | 1149.56    0.0    226     0 |    0.31   0.29  28.76   0.02
        2  1.01e+06 1928.41 | 1721.60    0.0    345     0 |    0.31   0.28  28.38   0.02
        2  1.02e+06 1928.41 | 1185.92    0.0    223     0 |    0.32   0.29  27.91   0.02
        2  1.03e+06 1928.41 | 1233.72    0.0    244     0 |    0.31   0.28  28.91   0.02
        2  1.04e+06 1928.41 | 1551.67    0.0    303     0 |    0.31   0.28  29.68   0.02
        2  1.05e+06 1928.41 |  978.83    0.0    192     0 |    0.31   0.28  28.76   0.02
        2  1.06e+06 2257.10 |
        2  1.06e+06 2257.10 | 2257.10  674.7    421   120 |    0.31   0.28  29.41   0.02
        2  1.07e+06 2257.10 | 2150.05    0.0    401     0 |    0.32   0.28  29.53   0.02
        2  1.08e+06 2257.10 |  775.42    0.0    149     0 |    0.31   0.28  29.71   0.02
        2  1.09e+06 2257.10 |  681.05    0.0    134     0 |    0.31   0.28  29.51   0.02
        2  1.10e+06 2257.10 | 1053.45    0.0    209     0 |    0.31   0.28  28.97   0.02
        2  1.11e+06 2257.10 |  850.63    0.0    170     0 |    0.31   0.27  29.28   0.02
        2  1.12e+06 2257.10 | 1736.51    0.0    344     0 |    0.31   0.28  30.47   0.02
        2  1.13e+06 2257.10 | 1936.67    0.0    370     0 |    0.32   0.27  29.64   0.02
        2  1.14e+06 2257.10 | 2247.72 1388.4    426   262 |    0.31   0.26  30.76   0.02
        2  1.15e+06 2825.53 |
        2  1.15e+06 2825.53 | 2825.53 1191.8    556   232 |    0.32   0.26  30.33   0.02
        2  1.16e+06 2825.53 | 2546.58 1666.1    498   332 |    0.32   0.27  30.56   0.02
        2  1.17e+06 2825.53 |  850.81    0.0    166     0 |    0.32   0.26  29.23   0.01
        2  1.18e+06 2825.53 | 1411.25    0.0    280     0 |    0.32   0.26  28.02   0.01
        2  1.19e+06 2825.53 |  538.52    0.0    109     0 |    0.32   0.26  31.44   0.01
        2  1.19e+06 2825.53 | 2693.81    0.0    508     0 |    0.31   0.26  30.05   0.01
        2  1.20e+06 2978.43 |
        2  1.20e+06 2978.43 | 2978.43 1663.4    573   317 |    0.32   0.25  30.95   0.01
        2  1.20e+06 3575.62 |
        2  1.20e+06 3575.62 | 3575.62 1612.5    694   314 |    0.31   0.26  31.57   0.01
        2  1.21e+06 3575.62 |  993.93    0.0    186     0 |    0.32   0.25  29.87   0.01
        2  1.21e+06 3575.62 | 1999.92    0.0    392     0 |    0.31   0.26  30.52   0.01
        2  1.22e+06 3575.62 |  889.42    0.0    171     0 |    0.33   0.25  30.81   0.01
        2  1.22e+06 3575.62 | 3405.88 1731.5    668   338 |    0.32   0.25  31.85   0.01
        2  1.23e+06 3575.62 |  892.11    0.0    168     0 |    0.32   0.25  31.85   0.01
        2  1.23e+06 3575.62 |  778.46    0.0    147     0 |    0.32   0.25  31.28   0.01
        2  1.24e+06 3575.62 | 1535.40    0.0    310     0 |    0.32   0.25  31.43   0.01
        2  1.24e+06 3575.62 | 2843.25 1210.4    554   243 |    0.32   0.25  29.70   0.01
        2  1.25e+06 3575.62 | 3353.49    0.0    631     0 |    0.31   0.25  30.64   0.01
        2  1.25e+06 3575.62 |  883.32    0.0    170     0 |    0.32   0.25  31.49   0.01
        2  1.26e+06 3575.62 | 3092.77 1210.6    606   226 |    0.32   0.25  31.16   0.01
        2  1.26e+06 3575.62 |  569.80    0.0    112     0 |    0.32   0.24  31.06   0.01
        2  1.27e+06 3575.62 | 1041.53    0.0    197     0 |    0.31   0.24  31.33   0.01
        2  1.28e+06 3575.62 | 3255.92    0.0    638     0 |    0.33   0.24  30.84   0.01
        2  1.28e+06 3575.62 | 2279.13    0.0    434     0 |    0.33   0.24  33.50   0.01
        2  1.29e+06 3575.62 |  559.01    0.0    107     0 |    0.32   0.24  32.44   0.01
        2  1.29e+06 3575.62 | 3277.19    0.0    649     0 |    0.32   0.25  32.05   0.01
        2  1.29e+06 3575.62 | 1050.02    0.0    204     0 |    0.33   0.24  31.84   0.01
        2  1.30e+06 3575.62 |  715.11    0.0    138     0 |    0.32   0.24  32.69   0.01
        2  1.31e+06 3575.62 |  569.46    0.0    111     0 |    0.32   0.24  31.83   0.01
        2  1.31e+06 3575.62 |  948.73    0.0    180     0 |    0.33   0.25  32.41   0.01
        2  1.31e+06 3575.62 | 1140.72    0.0    224     0 |    0.33   0.24  30.14   0.01
        2  1.32e+06 3575.62 | 1303.37    0.0    249     0 |    0.32   0.24  30.97   0.01
        2  1.32e+06 3575.62 | 2585.92    0.0    511     0 |    0.33   0.24  30.48   0.01
        2  1.33e+06 4376.07 |
        2  1.33e+06 4376.07 | 4376.07 1031.6    854   199 |    0.33   0.24  31.88   0.01
        2  1.34e+06 4376.07 | 2329.53    0.0    447     0 |    0.32   0.24  31.45   0.01
        2  1.34e+06 4376.07 | 2766.31 1240.2    565   254 |    0.32   0.24  32.76   0.01
        2  1.35e+06 4376.07 | 3559.24    0.0    672     0 |    0.32   0.24  32.08   0.01
        2  1.35e+06 4376.07 | 2982.04 1761.5    557   330 |    0.31   0.24  31.67   0.01
        2  1.36e+06 4376.07 | 1431.25    0.0    282     0 |    0.32   0.23  31.13   0.01
        2  1.36e+06 4376.07 | 1818.30    0.0    373     0 |    0.32   0.24  30.68   0.01
        2  1.37e+06 4376.07 |  911.64    0.0    167     0 |    0.33   0.24  31.94   0.01
        2  1.37e+06 4376.07 | 2429.27    0.0    442     0 |    0.32   0.24  31.20   0.01
        2  1.38e+06 4376.07 | 1491.57    0.0    294     0 |    0.33   0.23  31.37   0.01
        2  1.38e+06 4376.07 | 1535.84    0.0    294     0 |    0.33   0.22  32.09   0.01
        2  1.39e+06 4376.07 | 1724.81    0.0    331     0 |    0.33   0.24  31.07   0.01
        2  1.39e+06 4376.07 | 1426.36    0.0    273     0 |    0.33   0.23  32.00   0.01
        2  1.40e+06 4376.07 | 2066.45 1369.6    401   264 |    0.33   0.23  33.17   0.01
        2  1.40e+06 4376.07 | 1078.23    0.0    210     0 |    0.33   0.23  31.05   0.01
        2  1.41e+06 4376.07 | 1217.95    0.0    256     0 |    0.33   0.23  32.79   0.01
        2  1.41e+06 4376.07 | 1394.94    0.0    270     0 |    0.33   0.23  33.30   0.01
        2  1.42e+06 4376.07 | 3662.75    0.0    688     0 |    0.33   0.23  34.01   0.01
        2  1.42e+06 4376.07 | 3106.36    0.0    595     0 |    0.32   0.23  32.38   0.01
        2  1.43e+06 4376.07 | 3703.49    0.0    705     0 |    0.33   0.23  33.98   0.01
        2  1.43e+06 4376.07 | 1284.99    0.0    240     0 |    0.33   0.22  33.41   0.01
        2  1.44e+06 4376.07 | 2499.82    0.0    479     0 |    0.33   0.23  31.52   0.01
        2  1.44e+06 4376.07 | 4095.95 1608.4    780   308 |    0.33   0.22  31.57   0.01
        2  1.45e+06 4376.07 | 1376.74    0.0    268     0 |    0.33   0.22  34.45   0.01
        2  1.45e+06 4376.07 | 1529.86    0.0    293     0 |    0.33   0.23  33.78   0.01
        2  1.46e+06 4376.07 | 2346.18    0.0    472     0 |    0.33   0.23  33.08   0.01
        2  1.46e+06 4376.07 | 1454.02    0.0    278     0 |    0.33   0.22  31.71   0.01
        2  1.47e+06 4376.07 | 4071.25    0.0    775     0 |    0.32   0.22  32.64   0.01
        2  1.47e+06 4376.07 | 4041.26 1121.8    774   226 |    0.33   0.23  32.98   0.01
        2  1.48e+06 4376.07 | 2339.81    0.0    470     0 |    0.33   0.23  34.03   0.01
        2  1.48e+06 4376.07 | 3188.20 1323.7    610   249 |    0.33   0.22  31.73   0.01
        2  1.49e+06 4376.07 | 1301.07    0.0    270     0 |    0.33   0.22  33.16   0.01
        2  1.49e+06 4376.07 | 2600.90 1531.0    535   298 |    0.33   0.22  32.24   0.01
        2  1.50e+06 4376.07 | 2169.64    0.0    433     0 |    0.32   0.22  32.96   0.01
        2  1.51e+06 4376.07 | 1137.47    0.0    221     0 |    0.33   0.22  34.14   0.01
        2  1.51e+06 4376.07 | 3703.59 1616.3    704   305 |    0.33   0.22  32.40   0.01
        2  1.52e+06 4376.07 | 2139.60    0.0    406     0 |    0.33   0.21  32.47   0.01
        2  1.52e+06 4376.07 | 4352.19 1187.4    811   229 |    0.34   0.22  33.02   0.01
        2  1.53e+06 4376.07 | 2650.12 1653.3    486   297 |    0.33   0.22  33.92   0.01
        2  1.53e+06 4376.07 | 1533.19    0.0    303     0 |    0.34   0.21  31.45   0.01
        2  1.54e+06 4376.07 |  780.99    0.0    146     0 |    0.33   0.21  32.48   0.01
        2  1.54e+06 4376.07 | 2626.05    0.0    509     0 |    0.33   0.21  33.91   0.01
        2  1.55e+06 4376.07 | 4370.40    0.0    823     0 |    0.33   0.21  31.92   0.01
        2  1.55e+06 4376.07 | 1506.31    0.0    297     0 |    0.33   0.22  33.49   0.01
        2  1.56e+06 4376.07 | 2899.16 1496.5    554   282 |    0.33   0.21  32.71   0.01
        2  1.56e+06 4376.07 | 1198.79    0.0    236     0 |    0.34   0.20  34.19   0.01
        2  1.57e+06 4376.07 | 4314.28 1905.6    795   350 |    0.33   0.21  32.53   0.01
        2  1.57e+06 4376.07 |  932.14    0.0    185     0 |    0.33   0.20  32.61   0.01
        2  1.58e+06 4376.07 | 3710.02    0.0    680     0 |    0.33   0.21  33.56   0.01
        2  1.58e+06 4376.07 | 1788.02    0.0    334     0 |    0.33   0.20  32.80   0.01
        2  1.59e+06 4376.07 | 2178.98    0.0    405     0 |    0.33   0.20  35.01   0.01
        2  1.59e+06 4661.43 |
        2  1.59e+06 4661.43 | 4661.43 1205.4    870   225 |    0.33   0.21  34.16   0.01
        2  1.60e+06 4661.43 | 4636.75 1363.6    856   249 |    0.34   0.20  32.75   0.01
        2  1.60e+06 4661.43 | 3285.64    0.0    641     0 |    0.34   0.20  34.36   0.01
        2  1.61e+06 4661.43 | 3996.92    0.0    746     0 |    0.33   0.20  32.96   0.01
        2  1.61e+06 4808.85 |
        2  1.61e+06 4808.85 | 4808.85  858.2    905   165 |    0.32   0.20  32.36   0.01
        2  1.62e+06 4808.85 | 3711.65 1420.6    712   269 |    0.33   0.20  33.59   0.01
        2  1.62e+06 4808.85 | 2343.89    0.0    433     0 |    0.32   0.20  33.83   0.01
        2  1.63e+06 4808.85 | 3650.32 1767.0    675   326 |    0.33   0.19  33.05   0.01
        2  1.64e+06 4808.85 | 1539.43    0.0    279     0 |    0.33   0.19  34.10   0.01
        2  1.64e+06 4808.85 | 3677.19 1948.3    664   352 |    0.34   0.19  34.59   0.01
        2  1.65e+06 4808.85 | 2687.31    0.0    496     0 |    0.33   0.20  33.47   0.01
        2  1.65e+06 4808.85 | 1542.33    0.0    290     0 |    0.33   0.19  33.24   0.01
        2  1.66e+06 4808.85 |  867.88    0.0    173     0 |    0.34   0.19  34.27   0.01
        2  1.66e+06 4808.85 | 3085.34 1711.3    579   322 |    0.34   0.20  33.70   0.01
        2  1.67e+06 4808.85 | 1687.58    0.0    323     0 |    0.33   0.20  34.93   0.01
        2  1.67e+06 4808.85 | 4066.38 1268.1    742   228 |    0.34   0.19  32.41   0.01
        2  1.68e+06 4808.85 |  452.65    0.0     91     0 |    0.33   0.20  33.70   0.01
        2  1.68e+06 4808.85 | 3018.52    0.0    558     0 |    0.34   0.21  34.58   0.01
        2  1.69e+06 4808.85 |  430.01    0.0    102     0 |    0.33   0.35  38.89   0.02
        2  1.69e+06 4808.85 |  472.74    0.0    109     0 |    0.33   0.40  44.93   0.03
        2  1.70e+06 4808.85 |  628.25    0.0    136     0 |    0.31   0.32  44.62   0.02
        2  1.70e+06 4808.85 |  812.73    0.0    167     0 |    0.28   0.27  42.12   0.01
        2  1.70e+06 4808.85 | 1707.97    0.0    327     0 |    0.31   0.24  39.53   0.01
        2  1.71e+06 4808.85 | 1819.14    0.0    339     0 |    0.32   0.23  36.88   0.01
        2  1.72e+06 4808.85 | 2053.70    0.0    388     0 |    0.33   0.21  35.72   0.01
        2  1.72e+06 4808.85 | 2262.07    0.0    419     0 |    0.33   0.21  33.69   0.01
        2  1.72e+06 4808.85 |  787.00    0.0    153     0 |    0.33   0.20  35.14   0.01
        2  1.73e+06 4808.85 | 1620.02    0.0    309     0 |    0.34   0.20  33.92   0.01
        2  1.74e+06 4808.85 | 2368.23    0.0    447     0 |    0.33   0.20  34.85   0.01
        2  1.74e+06 4846.42 |
        2  1.74e+06 4846.42 | 4846.42  581.3    906   110 |    0.33   0.19  34.40   0.01
        2  1.75e+06 4846.42 |  752.17    0.0    150     0 |    0.34   0.19  33.20   0.01
        2  1.75e+06 4846.42 | 3085.69 2158.1    576   395 |    0.34   0.19  34.28   0.01
        2  1.76e+06 4846.42 |  381.14    0.0     71     0 |    0.34   0.20  33.92   0.01
        2  1.76e+06 4846.42 | 3684.18    0.0    693     0 |    0.34   0.19  34.47   0.01
        2  1.77e+06 5436.18 |
        2  1.77e+06 5436.18 | 5436.18   35.3   1000     0 |    0.33   0.19  34.15   0.01
        2  1.77e+06 5436.18 | 5327.19    0.0   1000     0 |    0.34   0.19  33.98   0.01
        2  1.78e+06 5436.18 | 4729.78    0.0    857     0 |    0.34   0.20  34.32   0.01
        2  1.78e+06 5578.46 |
        2  1.78e+06 5578.46 | 5578.46   86.6   1000     0 |    0.34   0.19  34.41   0.01
        2  1.79e+06 5578.46 | 2561.99    0.0    473     0 |    0.34   0.19  34.08   0.01
        2  1.79e+06 5578.46 | 5432.93    0.0   1000     0 |    0.35   0.19  34.19   0.01
        2  1.80e+06 5578.46 | 1247.43    0.0    232     0 |    0.34   0.19  34.27   0.01
        2  1.80e+06 5578.46 | 3227.15    0.0    598     0 |    0.35   0.18  35.06   0.01
        2  1.81e+06 5578.46 | 3294.27    0.0    610     0 |    0.35   0.18  34.52   0.01
        2  1.81e+06 5578.46 | 5424.16    0.0   1000     0 |    0.33   0.18  34.81   0.01
        2  1.82e+06 5578.46 | 5410.13    0.0   1000     0 |    0.34   0.18  34.04   0.01
        2  1.82e+06 5578.46 | 2406.91    0.0    454     0 |    0.34   0.18  35.67   0.01
        2  1.83e+06 5578.46 | 5494.68    0.0   1000     0 |    0.34   0.18  35.15   0.01
        2  1.83e+06 5578.46 | 2468.08    0.0    535     0 |    0.34   0.19  34.65   0.01
        2  1.84e+06 5578.46 | 3286.31    0.0    633     0 |    0.34   0.19  34.83   0.01
        2  1.84e+06 5578.46 | 5295.30    0.0   1000     0 |    0.33   0.19  33.75   0.01
        2  1.85e+06 5578.46 | 1098.72    0.0    216     0 |    0.34   0.18  35.00   0.01
        2  1.85e+06 5578.46 | 2036.09    0.0    381     0 |    0.34   0.18  34.08   0.01
        2  1.86e+06 5578.46 |  704.63    0.0    135     0 |    0.34   0.18  32.72   0.01
        2  1.86e+06 5578.46 | 5437.36    0.0   1000     0 |    0.33   0.18  35.17   0.01
        2  1.87e+06 5578.46 | 1882.41    0.0    352     0 |    0.34   0.18  33.29   0.01
        2  1.87e+06 5578.46 | 2585.14    0.0    453     0 |    0.34   0.18  34.44   0.01
        2  1.88e+06 5578.46 | 3676.13    0.0    674     0 |    0.34   0.18  33.99   0.01
        2  1.88e+06 5578.46 | 2395.74    0.0    438     0 |    0.34   0.18  34.04   0.01
        2  1.89e+06 5578.46 | 1226.04    0.0    241     0 |    0.34   0.17  34.84   0.01
        2  1.89e+06 5578.46 | 5255.36    0.0   1000     0 |    0.34   0.18  33.56   0.01
        2  1.90e+06 5578.46 | 5571.49    0.0   1000     0 |    0.35   0.17  34.05   0.01
        2  1.90e+06 5578.46 | 3593.36    0.0    716     0 |    0.34   0.18  33.59   0.01
        2  1.91e+06 5578.46 | 5293.41    0.0   1000     0 |    0.35   0.18  34.38   0.01
        2  1.91e+06 5578.46 | 4169.86    0.0    795     0 |    0.33   0.18  34.22   0.01
        2  1.92e+06 5578.46 | 3631.33    0.0    647     0 |    0.34   0.17  34.96   0.01
        2  1.92e+06 5578.46 | 5460.01    0.0   1000     0 |    0.35   0.17  34.32   0.01
        2  1.93e+06 5578.46 | 5410.56    0.0   1000     0 |    0.35   0.17  33.34   0.01
        2  1.93e+06 5578.46 | 5460.13    0.0   1000     0 |    0.34   0.17  33.90   0.01
        2  1.94e+06 5578.46 | 2817.41    0.0    537     0 |    0.34   0.18  34.36   0.01
        2  1.94e+06 5578.46 | 1858.92    0.0    339     0 |    0.35   0.17  34.96   0.01
        2  1.95e+06 5578.46 | 2207.20    0.0    418     0 |    0.34   0.17  35.02   0.01
        2  1.95e+06 5578.46 | 2759.04    0.0    523     0 |    0.35   0.17  35.04   0.01
        2  1.96e+06 5578.46 | 5266.78    0.0    940     0 |    0.35   0.17  33.96   0.01
        2  1.96e+06 5578.46 | 1092.09    0.0    203     0 |    0.34   0.18  33.64   0.01
        2  1.97e+06 5578.46 | 2487.55    0.0    446     0 |    0.34   0.26  37.81   0.02
        2  1.97e+06 5578.46 | 2313.96    0.0    420     0 |    0.34   0.23  40.44   0.01
        2  1.98e+06 5606.16 |
        2  1.98e+06 5606.16 | 5606.16   51.8   1000     0 |    0.32   0.21  38.85   0.01
        2  1.98e+06 5606.16 | 3925.41    0.0    696     0 |    0.34   0.19  37.93   0.01
        2  1.99e+06 5606.16 | 5475.95    0.0   1000     0 |    0.35   0.18  35.79   0.01
        2  1.99e+06 5606.16 | 2794.34    0.0    516     0 |    0.36   0.19  34.80   0.01
        2  2.00e+06 5606.16 | 5602.57    0.0   1000     0 |    0.35   0.17  34.41   0.01
        2  2.00e+06 5606.16 | 5382.77  326.0    968    56 |    0.35   0.18  35.26   0.01
        | UsedTime:   45183 | SavedDir: ./Humanoid-v3_ReliableSAC_2
        | Learner: Save in ./Humanoid-v3_ReliableSAC_2
        """
    else:
        raise ValueError('env_name:', env_name)

    args.learner_gpus = gpu_id
    args.random_seed += gpu_id

    if_check = 0
    if if_check:
        train_and_evaluate(args)
    else:
        train_and_evaluate_mp(args)


def demo_ddpg_h_term(gpu_id, drl_id, env_id):  # 2022.04.04
    env_name = ['Pendulum-v1',
                'BipedalWalker-v3',
                'Hopper-v2',
                'HalfCheetah-v3',
                'Walker2d-v3',
                'Humanoid-v3', ][env_id]
    agent_class = [AgentDDPG, AgentDDPGHterm, AgentTD3,
                   AgentSAC, AgentReSAC, AgentReSACHterm, AgentReSACHtermK][drl_id]

    # from elegantrl.train.config import get_gym_env_args
    # env = gym.make(env_name)
    # get_gym_env_args(env=env, if_print=True)
    # exit()

    if env_name in {'Pendulum-v0', 'Pendulum-v1'}:
        from elegantrl.envs.CustomGymEnv import PendulumEnv
        env = PendulumEnv(env_name, target_return=-500)

        args = Arguments(agent_class, env)
        args.reward_scale = 2 ** -1  # RewardRange: -1800 < -200 < -50 < 0
        args.gamma = 0.95
        args.target_step = args.max_step * 4
        args.eval_times = 2 ** 5

        """
        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        2  1.60e+03-1147.49 |-1147.49  179.2    200     0 |   -2.61   0.90   0.55   1.00
        2  5.84e+04 -121.61 | -121.61   59.0    200     0 |   -0.81   0.33 -40.64   0.79
        | UsedTime:     132 |

        ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
        1  1.60e+03-1267.96 |-1267.96  329.7    200     0 |   -2.67   0.88   0.56   1.00
        1  8.48e+04 -171.79 | -182.24   63.3    200     0 |   -0.30   0.32 -30.75   0.64
        1  1.19e+05 -171.79 | -178.25  116.8    200     0 |   -0.31   0.16 -22.52   0.43
        1  1.34e+05 -164.56 | -164.56   99.1    200     0 |   -0.31   0.15 -18.09   0.35
        1  1.47e+05 -135.20 | -135.20   92.1    200     0 |   -0.31   0.14 -15.65   0.29
        | UsedTime:     783 | 
        """
    elif env_name == 'BipedalWalker-v3':
        env_func = gym.make
        env_args = {'env_num': 1,
                    'env_name': 'BipedalWalker-v3',
                    'max_step': 1600,
                    'state_dim': 24,
                    'action_dim': 4,
                    'if_discrete': False,
                    'target_return': 300,

                    'id': 'BipedalWalker-v3', }
        args = Arguments(agent_class, env_func=env_func, env_args=env_args)
        args.reward_scale = 2 ** -1
        args.target_step = args.max_step

        args.gamma = 0.98
        args.eval_times = 2 ** 4

        """
        | Arguments Remove cwd: ./BipedalWalker-v3_ReliableSAC_5
################################################################################
ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
5  1.78e+03 -111.40 |
5  1.78e+03 -111.40 | -111.40    0.2    102     4 |   -0.18   0.75   1.25   0.25
5  6.72e+04 -102.81 |
5  6.72e+04 -102.81 | -102.81    0.6     56     2 |   -0.40   0.13   4.46   0.03
5  1.02e+05  -57.47 |
5  1.02e+05  -57.47 |  -57.47    9.6   1600     0 |   -0.15   0.41  -4.42   0.03
5  1.24e+05  -45.19 |
5  1.24e+05  -45.19 |  -45.19    8.3   1600     0 |   -0.04   0.46  -4.50   0.04
5  1.44e+05  -33.45 |
5  1.44e+05  -33.45 |  -33.45    3.0   1600     0 |   -0.40   0.26  -3.91   0.03
5  1.61e+05  -33.45 |  -40.02   11.3   1600     0 |   -0.03   0.13  -2.09   0.02
5  1.78e+05  -33.45 |  -47.55   10.9   1600     0 |   -0.37   0.14   0.28   0.02
5  1.96e+05  -33.45 |  -94.23    0.5    147     5 |   -0.11   0.11  -2.00   0.03
5  2.13e+05  108.48 |
5  2.13e+05  108.48 |  108.48  118.5   1248   566 |    0.05   0.12  -2.35   0.03
5  2.28e+05  108.48 |   54.40  123.4   1057   553 |   -0.45   0.14   0.47   0.03
5  2.41e+05  258.69 |
5  2.41e+05  258.69 |  258.69   91.5   1371   283 |    0.06   0.13   1.60   0.04
5  2.58e+05  299.87 |
5  2.58e+05  299.87 |  299.87    0.9   1443    19 |    0.09   0.15   0.31   0.04
5  2.74e+05  299.87 |  243.41  113.8   1119   295 |    0.16   0.13   3.68   0.04
5  2.86e+05  299.87 |   80.06  125.7    635   294 |   -0.22   0.11   4.27   0.04
5  2.98e+05  312.25 |
5  2.98e+05  312.25 |  312.25    1.0   1186    16 |    0.22   0.13   5.49   0.04
| UsedTime: 1885 | SavedDir: ./BipedalWalker-v3_ReliableSAC_5

        
        | Arguments Remove cwd: ./BipedalWalker-v3_ReliableSACHterm_6
################################################################################
ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
6  1.60e+03  -92.22 |
6  1.60e+03  -92.22 |  -92.22    0.1    110     1 |   -0.06   0.74   1.15   0.25
6  6.50e+04  -31.14 |
6  6.50e+04  -31.14 |  -31.14    1.7   1600     0 |   -0.04   0.08   4.60   0.02
6  9.71e+04  -31.14 | -103.20    0.1     98     1 |   -0.44   0.09  -0.19   0.02
6  1.24e+05  -31.14 |  -62.62   29.0   1304   593 |   -0.05   0.19  -3.96   0.03
6  1.45e+05  -31.14 |  -39.96   12.3   1600     0 |   -0.19   0.24  -6.13   0.02
6  1.65e+05  -31.14 |  -45.63   15.5   1600     0 |   -0.11   0.20   0.42   0.02
6  1.82e+05  -31.14 |  -55.53    7.1   1600     0 |   -0.04   0.12  -1.60   0.02
6  1.98e+05  -31.14 |  -88.54   33.1    686   746 |    0.04   0.08  -1.20   0.02
6  2.14e+05  -31.14 |  -45.76   29.0    759   267 |   -0.11   0.11   0.02   0.03
6  2.29e+05  249.59 |
6  2.29e+05  249.59 |  249.59  105.0   1436   375 |   -0.18   0.16   1.73   0.04
6  2.42e+05  249.59 |   12.51   82.6    559   379 |    0.13   0.20   1.16   0.04
6  2.60e+05  309.47 |
6  2.60e+05  309.47 |  309.47    1.6   1455    33 |    0.07   0.18   1.84   0.04
| UsedTime: 1476 | SavedDir: ./BipedalWalker-v3_ReliableSACHterm_6

        """
    elif env_name == 'Hopper-v2':
        env_func = gym.make
        env_args = {
            'env_num': 1,
            'env_name': 'Hopper-v2',
            'max_step': 1000,
            'state_dim': 11,
            'action_dim': 3,
            'if_discrete': False,
            'target_return': 3800.,
        }
        args = Arguments(agent_class, env_func=env_func, env_args=env_args)

        args.num_layer = 3
        args.net_dim = 2 ** 8
        args.batch_size = int(args.net_dim * 2)

        args.worker_num = 2
        args.target_step = args.max_step
        args.repeat_times = 2 ** -1
        args.reward_scale = 2 ** -4

        args.learning_rate = 2 ** -15
        args.clip_grad_norm = 1.0
        args.lambda_entropy = 0.05
        args.gamma = 0.993

        args.if_act_target = False
        args.explore_noise_std = 0.1  # for DPG

        args.h_term_sample_rate = 2 ** -2
        args.h_term_drop_rate = 2 ** -3
        args.h_term_lambda = 2 ** -6
        args.act_update_gap = 1
        args.h_term_k_step = 4
        args.h_term_update_gap = 2

        args.eval_times = 2 ** 1
        args.eval_gap = 2 ** 8
        args.if_allow_break = False
        args.break_step = int(2e6)

        """
        ;;; repeat_times 0.5
;;; explore_noise_std 0.06
| Arguments Remove cwd: ./Hopper-v2_DDPG_5
################################################################################
ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
5  4.05e+03   31.31 |
5  4.05e+03   31.31 |   31.31    0.6     33     0 |    0.06   0.00  -0.15
5  2.60e+05  125.04 |
5  2.60e+05  125.04 |  125.04   36.8     72    19 |    0.11   0.05  -4.96
5  4.52e+05  287.91 |
5  4.52e+05  287.91 |  287.91    9.1    150     7 |    0.10   0.10 -15.62
5  6.04e+05  287.91 |   33.76    0.0     23     0 |    0.11   0.33 -29.86
5  7.28e+05  943.68 |
5  7.28e+05  943.68 |  943.68    2.5   1000     0 |    0.05   0.20 -36.74
5  8.39e+05  943.68 |  223.96    0.0    276     0 |    0.10   0.39 -32.80
5  9.36e+05  943.68 |   18.41    0.0     17     0 |    0.11   0.28 -47.11
5  1.02e+06  943.68 |  112.46    0.0     79     0 |    0.10   0.43 -44.61
5  1.11e+06  943.68 |  271.21    0.0    139     0 |    0.11   0.15 -43.29
5  1.19e+06  943.68 |  339.68    0.0    173     0 |    0.12   0.26 -44.58
5  1.26e+06  943.68 |  344.69    0.0    151     0 |    0.12   0.80 -68.53
5  1.33e+06  943.68 |  246.59    0.0    150     0 |    0.18   0.47 -63.54
5  1.39e+06  943.68 |  520.48    0.0    379     0 |    0.09   1.04 -69.88
5  1.46e+06 1515.81 |
5  1.46e+06 1515.81 | 1515.81  561.6    538   271 |    0.14   0.54 -59.63
5  1.53e+06 1515.81 |  287.81    0.0    119     0 |    0.17   0.54 -55.94
5  1.59e+06 1515.81 |  317.89    0.0    124     0 |    0.18   0.48 -54.38
5  1.65e+06 1515.81 |  777.02    0.0    255     0 |    0.18   0.52 -53.06
5  1.70e+06 1515.81 |  260.43    0.0    135     0 |    0.13   0.37 -48.36
5  1.76e+06 1515.81 |  856.31    0.0    252     0 |    0.19   0.63 -47.88
5  1.82e+06 1515.81 | 1049.09    0.0    360     0 |    0.17   0.41 -48.26
5  1.87e+06 1515.81 | 1248.30    0.0    357     0 |    0.20   0.58 -53.84
5  1.92e+06 3040.54 |
5  1.92e+06 3040.54 | 3040.54   60.8   1000     0 |    0.19   0.25 -51.93
5  1.97e+06 3040.54 |  384.53    0.0    149     0 |    0.13   0.24 -47.11
| UsedTime:    2952 | SavedDir: ./Hopper-v2_DDPG_5
| Learner: Save in ./Hopper-v2_DDPG_5

        
        ;;; repeat_times 0.5
| Arguments Remove cwd: ./Hopper-v2_ReliableSAC_5
################################################################################
ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
5  4.02e+03   22.57 |
5  4.02e+03   22.57 |   22.57    0.5     28     0 |    0.05   0.81   1.37   0.33
5  1.93e+05  264.67 |
5  1.93e+05  264.67 |  264.67    2.3    111     1 |    0.15   0.10  12.14   0.09
5  3.04e+05  419.28 |
5  3.04e+05  419.28 |  419.28    1.3    142     0 |    0.16   0.12  19.81   0.11
5  3.91e+05  679.15 |
5  3.91e+05  679.15 |  679.15    3.2    246     2 |    0.18   0.16  28.57   0.14
5  4.68e+05 2296.08 |
5  4.68e+05 2296.08 | 2296.08   23.0   1000     0 |    0.14   0.18  33.53   0.15
5  5.38e+05 2296.08 | 1975.77    0.0   1000     0 |    0.13   0.19  35.84   0.14
5  6.00e+05 2513.52 |
5  6.00e+05 2513.52 | 2513.52    2.8   1000     0 |    0.17   0.18  39.30   0.13
5  6.51e+05 2627.30 |
5  6.51e+05 2627.30 | 2627.30    0.6   1000     0 |    0.16   0.17  40.20   0.12
5  7.00e+05 2627.30 | 2565.19    0.0   1000     0 |    0.16   0.15  40.18   0.11
5  7.44e+05 2820.52 |
5  7.44e+05 2820.52 | 2820.52    1.8   1000     0 |    0.17   0.15  40.12   0.09
5  7.88e+05 2820.52 | 2597.41    0.0   1000     0 |    0.17   0.13  39.35   0.08
5  8.28e+05 2820.52 | 2795.26    0.0   1000     0 |    0.18   0.11  38.10   0.07
5  8.69e+05 2936.57 |
5  8.69e+05 2936.57 | 2936.57    2.1   1000     0 |    0.18   0.11  37.18   0.07
5  9.05e+05 3017.49 |
5  9.05e+05 3017.49 | 3017.49    1.2   1000     0 |    0.19   0.10  36.89   0.06
5  9.41e+05 3017.49 | 2869.90    0.0   1000     0 |    0.19   0.09  35.58   0.06
5  9.80e+05 3079.65 |
5  9.80e+05 3079.65 | 3079.65    2.1   1000     0 |    0.19   0.09  35.00   0.06
5  1.01e+06 3111.62 |
5  1.01e+06 3111.62 | 3111.62    1.2   1000     0 |    0.20   0.09  35.51   0.06
5  1.04e+06 3111.62 | 3102.43    0.0   1000     0 |    0.19   0.09  35.28   0.06
5  1.08e+06 3195.77 |
5  1.08e+06 3195.77 | 3195.77    0.8   1000     0 |    0.20   0.08  35.27   0.06
5  1.10e+06 3204.82 |
5  1.10e+06 3204.82 | 3204.82    1.3   1000     0 |    0.20   0.08  35.29   0.05
5  1.13e+06 3227.59 |
5  1.13e+06 3227.59 | 3227.59    0.3   1000     0 |    0.20   0.08  34.81   0.05
5  1.17e+06 3236.78 |
5  1.17e+06 3236.78 | 3236.78    1.1   1000     0 |    0.20   0.07  34.90   0.05
5  1.19e+06 3251.62 |
5  1.19e+06 3251.62 | 3251.62    0.6   1000     0 |    0.20   0.07  34.88   0.05
5  1.22e+06 3310.67 |
5  1.22e+06 3310.67 | 3310.67    1.2   1000     0 |    0.21   0.07  34.24   0.05
5  1.25e+06 3310.67 | 3303.20    0.0   1000     0 |    0.21   0.07  34.15   0.05
5  1.28e+06 3312.09 |
5  1.28e+06 3312.09 | 3312.09    1.3   1000     0 |    0.20   0.07  33.99   0.05
5  1.31e+06 3361.26 |
5  1.31e+06 3361.26 | 3361.26    1.3   1000     0 |    0.20   0.07  34.49   0.05
5  1.34e+06 3361.26 | 3264.82    0.0   1000     0 |    0.21   0.07  34.20   0.05
5  1.36e+06 3361.26 | 3218.51    0.0   1000     0 |    0.22   0.07  34.42   0.05
5  1.39e+06 3361.26 | 3308.47    0.0   1000     0 |    0.22   0.06  33.95   0.05
5  1.41e+06 3361.26 | 3347.62    0.0   1000     0 |    0.22   0.06  34.16   0.05
5  1.44e+06 3361.26 | 3285.74    0.0   1000     0 |    0.21   0.07  34.00   0.05
5  1.46e+06 3361.26 | 3280.85    0.0   1000     0 |    0.22   0.06  33.90   0.04
5  1.49e+06 3361.26 | 3311.28    0.0   1000     0 |    0.21   0.06  34.01   0.04
5  1.51e+06 3407.79 |
5  1.51e+06 3407.79 | 3407.79    1.7   1000     0 |    0.22   0.06  33.94   0.05
5  1.54e+06 3407.79 | 3381.80    0.0   1000     0 |    0.20   0.06  34.44   0.05
5  1.56e+06 3407.79 | 3356.46    0.0   1000     0 |    0.22   0.06  34.72   0.05
5  1.59e+06 3407.79 | 3395.07    0.0   1000     0 |    0.20   0.06  34.86   0.05
5  1.61e+06 3407.79 | 3313.27    0.0   1000     0 |    0.22   0.06  34.64   0.05
5  1.63e+06 3407.79 | 3401.76    0.0   1000     0 |    0.22   0.06  34.94   0.05
5  1.66e+06 3489.30 |
5  1.66e+06 3489.30 | 3489.30    4.1   1000     0 |    0.23   0.06  34.42   0.05
5  1.68e+06 3489.30 | 3393.20    0.0   1000     0 |    0.23   0.06  34.14   0.05
5  1.70e+06 3489.30 | 3458.37    0.0   1000     0 |    0.22   0.05  34.41   0.05
5  1.73e+06 3489.30 | 3352.92    0.0   1000     0 |    0.22   0.06  34.62   0.05
5  1.75e+06 3489.30 | 3333.69    0.0   1000     0 |    0.22   0.06  35.26   0.05
5  1.77e+06 3489.30 | 3456.53    0.0   1000     0 |    0.21   0.05  35.47   0.05
| UsedTime:    5997 | SavedDir: ./Hopper-v2_ReliableSAC_5
| Learner: Save in ./Hopper-v2_ReliableSAC_5

| Arguments Remove cwd: ./Hopper-v2_ReliableSACHterm_6
################################################################################
ID     Step    maxR |    avgR   stdR   avgS  stdS |    expR   objC   etc.
6  4.04e+03   78.95 |
6  4.04e+03   78.95 |   78.95    3.6    108     4 |    0.09   0.76   0.85   0.33
6  1.15e+05  198.10 |
6  1.15e+05  198.10 |  198.10    1.9     90     1 |    0.28   0.63  29.79   0.23
6  1.72e+05  889.93 |
6  1.72e+05  889.93 |  889.93    4.0    314     1 |    0.28   0.82  45.62   0.17
6  2.17e+05  889.93 |  274.09    0.0    110     0 |    0.31   0.94  58.49   0.21
6  2.55e+05  889.93 |  491.19    0.0    167     0 |    0.34   1.29  58.00   0.28
6  2.91e+05  889.93 |  261.90    0.0    119     0 |    0.30   2.26  58.54   0.49
6  3.25e+05 1031.75 |
6  3.25e+05 1031.75 | 1031.75    0.5   1000     0 |    0.15   1.82  85.02   0.71
6  3.54e+05 1031.75 | 1014.32    0.0   1000     0 |    0.13   1.47 101.81   0.50
6  3.81e+05 1034.31 |
6  3.81e+05 1034.31 | 1034.31    0.6   1000     0 |    0.14   1.17  95.27   0.42
6  4.06e+05 1034.31 |  122.12    0.0    151     0 |    0.14   0.91  81.74   0.36
6  4.29e+05 1034.31 |  277.10    0.0    221     0 |    0.14   0.78  73.37   0.34
6  4.51e+05 1034.31 |  565.33    0.0    384     0 |    0.16   0.75  68.32   0.31
6  4.71e+05 1378.52 |
6  4.71e+05 1378.52 | 1378.52    0.1   1000     0 |    0.22   0.60  66.21   0.26
6  4.89e+05 1378.52 |  448.34    0.0    315     0 |    0.21   0.49  60.09   0.20
6  5.06e+05 1378.52 |  419.95    0.0    168     0 |    0.29   0.44  53.30   0.17
6  5.24e+05 1378.52 | 1104.66    0.0    420     0 |    0.32   0.35  47.74   0.17
6  5.41e+05 1649.15 |
6  5.41e+05 1649.15 | 1649.15  879.9    704   300 |    0.29   0.31  45.29   0.15
6  5.60e+05 2190.47 |
6  5.60e+05 2190.47 | 2190.47    4.3   1000     0 |    0.25   0.33  44.60   0.14
6  5.77e+05 2190.47 | 2120.56    0.0   1000     0 |    0.33   0.32  42.70   0.12
6  5.95e+05 2686.57 |
6  5.95e+05 2686.57 | 2686.57    5.8   1000     0 |    0.38   0.27  41.22   0.11
6  6.08e+05 2686.57 |  956.68    0.0    292     0 |    0.39   0.25  40.32   0.11
6  6.21e+05 2686.57 |  273.05    0.0    132     0 |    0.34   0.23  39.83   0.11
6  6.34e+05 2686.57 |  308.15    0.0    137     0 |    0.30   0.24  40.72   0.11
6  6.47e+05 2686.57 |  458.55    0.0    200     0 |    0.29   0.27  40.67   0.11
6  6.60e+05 2686.57 |  760.84    0.0    286     0 |    0.34   0.27  41.64   0.11
6  6.75e+05 2686.57 | 1253.21    0.0    412     0 |    0.35   0.23  41.98   0.11
6  6.88e+05 2686.57 |  544.23    0.0    205     0 |    0.33   0.24  41.93   0.11
6  7.01e+05 2686.57 | 2637.08    0.0   1000     0 |    0.37   0.23  42.04   0.11
6  7.16e+05 2686.57 | 1063.86    0.0    369     0 |    0.35   0.23  42.23   0.12
6  7.27e+05 3082.43 |
6  7.27e+05 3082.43 | 3082.43   94.6    967    30 |    0.38   0.23  44.18   0.12
6  7.38e+05 3082.43 | 2714.58    0.0   1000     0 |    0.36   0.23  43.15   0.11
6  7.50e+05 3082.43 | 1576.83    0.0    530     0 |    0.36   0.22  42.79   0.11
6  7.62e+05 3082.43 | 2904.31    0.0   1000     0 |    0.37   0.22  42.24   0.12
6  7.74e+05 3082.43 | 2907.68    0.0   1000     0 |    0.39   0.22  43.42   0.12
6  7.87e+05 3082.43 | 3047.61    0.0   1000     0 |    0.38   0.23  43.15   0.12
6  8.01e+05 3082.43 | 2884.98    0.0   1000     0 |    0.39   0.23  44.60   0.12
6  8.15e+05 3082.43 | 3010.91    0.0   1000     0 |    0.40   0.23  45.58   0.12
6  8.27e+05 3082.43 | 2904.95    0.0   1000     0 |    0.36   0.23  43.33   0.13
6  8.39e+05 3082.43 | 2962.38    0.0   1000     0 |    0.38   0.20  45.61   0.13
6  8.53e+05 3082.43 | 2986.82    0.0   1000     0 |    0.37   0.22  46.55   0.13
6  8.65e+05 3082.43 | 3012.73    0.0   1000     0 |    0.39   0.22  46.56   0.12
6  8.77e+05 3082.43 | 3006.50    0.0   1000     0 |    0.38   0.20  45.55   0.12
6  8.89e+05 3082.43 | 2963.94    0.0   1000     0 |    0.39   0.20  46.11   0.12
6  9.00e+05 3082.43 | 2965.68    0.0   1000     0 |    0.38   0.20  45.67   0.12
6  9.10e+05 3082.43 | 2978.31    0.0   1000     0 |    0.39   0.19  47.14   0.12
6  9.20e+05 3082.43 | 3036.60    0.0   1000     0 |    0.39   0.18  46.73   0.12
6  9.32e+05 3082.43 | 3030.23    0.0   1000     0 |    0.37   0.20  46.14   0.12
6  9.41e+05 3082.43 | 2977.58    0.0   1000     0 |    0.38   0.19  46.32   0.12
6  9.53e+05 3082.43 | 2947.05    0.0   1000     0 |    0.39   0.19  45.68   0.12
6  9.62e+05 3082.43 | 3048.31    0.0   1000     0 |    0.38   0.18  46.31   0.12
6  9.71e+05 3082.43 | 3021.77    0.0   1000     0 |    0.39   0.18  45.73   0.12
| UsedTime:    6967 | SavedDir: ./Hopper-v2_ReliableSACHterm_6

        """
    elif env_name == 'HalfCheetah-v3':
        env_func = gym.make
        env_args = {
            'env_num': 1,
            'env_name': 'HalfCheetah-v3',
            'max_step': 1000,
            'state_dim': 17,
            'action_dim': 6,
            'if_discrete': False,
            'target_return': 4800.0,
        }
        args = Arguments(agent_class, env_func=env_func, env_args=env_args)

        args.num_layer = 3
        args.net_dim = 2 ** 8
        args.batch_size = int(args.net_dim * 2)

        args.worker_num = 2
        args.target_step = args.max_step
        args.repeat_times = 2 ** -1
        args.reward_scale = 2 ** -3

        args.learning_rate = 2 ** -15
        args.clip_grad_norm = 1.0
        args.gamma = 0.99

        args.if_act_target = False
        args.explore_noise_std = 0.06  # for Deterministic Policy Gradient Algorithms

        args.h_term_sample_rate = 2 ** -2
        args.h_term_drop_rate = 2 ** -3
        args.h_term_lambda = 2 ** -6
        args.h_term_k_step = 4
        args.h_term_update_gap = 2

        args.eval_times = 2 ** 2
        args.eval_gap = 2 ** 8
        args.if_allow_break = False
        args.break_step = int(4e6)
    elif env_name == 'Walker2d-v3':
        env_func = gym.make
        env_args = {
            'env_num': 1,
            'env_name': 'Walker2d-v3',
            'if_discrete': False,
            'max_step': 1000,
            'state_dim': 17,
            'action_dim': 6,
            'target_return': 65536
        }
        args = Arguments(agent_class, env_func=env_func, env_args=env_args)

        args.num_layer = 3
        args.net_dim = 2 ** 7
        args.batch_size = int(args.net_dim * 2)

        args.worker_num = 2
        args.target_step = args.max_step
        args.repeat_times = 2 ** -1
        args.reward_scale = 2 ** -4

        args.learning_rate = 2 ** -15
        args.clip_grad_norm = 1.0
        args.gamma = 0.99

        args.if_act_target = False
        args.explore_noise_std = 0.1  # for DPG

        args.h_term_sample_rate = 2 ** -2
        args.h_term_drop_rate = 2 ** -3
        args.h_term_lambda = 2 ** -6
        args.h_term_k_step = 4
        args.h_term_update_gap = 2

        args.eval_times = 2 ** 1
        args.eval_gap = 2 ** 8
        args.if_allow_break = False
        args.break_step = int(2e6)
    else:
        raise ValueError('env_name:', env_name)

    args.learner_gpus = gpu_id
    args.random_seed += gpu_id + 455

    if_check = 0
    if if_check:
        train_and_evaluate(args)
    else:
        train_and_evaluate_mp(args)


if __name__ == '__main__':
    GPU_ID = int(sys.argv[1]) if len(sys.argv) > 1 else 0  # >=0 means GPU ID, -1 means CPU
    DRL_ID = int(sys.argv[2]) if len(sys.argv) > 2 else 1
    ENV_ID = int(sys.argv[3]) if len(sys.argv) > 3 else 0

    demo_ddpg_h_term(GPU_ID, DRL_ID, ENV_ID)
